truss, apptrace - 윈디하나의 솔라나라
|
stderr로 출력된다.truss(1)와 같은 유틸은
- 시스템 콜의 진입 시점에서, 실행중인 프로세스를 중단하고, 시스템 콜 호출 인자를 알아낸 후, 다시 프로세스를 실행한다.
- 시스템 콜의 반환 시점에서, 실행중인 프로세스를 중단하고, 시스템 콜 반환 내용을 알아낸 후, 다시 프로세스를 실행한다.
즉 추적하려하는 프로세스의 시스템 콜 정보를 얻기 위해 프로세스를 정지했다가 재시작(정확히는 컨텍스트 스위칭)하는 일을 반복한다.
단일 프로세스, 단일 쓰레드를 사용하는 경우에는 문제가 될 일이 없지만, 타이밍에 민감한 멀티 쓰레드 프로그램이나 실시간 프로세스는 큰 문제가 될 수 있다. 또한 64비트 프로세스에서는 더 느려진다. 이러한 문제는 DTrace에서 해결되었다. 윈디하나의 솔라나라: DTrace (작성중)를 읽어보자. 또한 DTrace 기반으로 truss 를 흉내낸 dtruss 라는 스크립트가 DTraceToolkit에 포함되어 배포되고 있으니 사용해보자.
truss [옵션] 명령exec(2), fork(2) 시스템 콜에 의해 생성된 프로세스까지 트레이스 하려면 -f 옵션을 같이 사용하면 된다.
truss [옵션] -p PID
-f옵션을 주어 실행해야 원하는 트레이싱이 가능하다.
truss_sample.c | (363 바이트) |
#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>
void main() {
printf("Wait for 10 secs\n");
sleep(10);
pid_t f = fork();
if (f < 0) {
printf("Unable to fork.\n");
} else if (f == 0) {
printf("Hello, World! I'm child. PID=%d\n", getpid());
} else {
printf("Hello, World! PID=%d\n", getpid());
}
}
root@wl ~ # cc -o truss_sample truss_sample.c root@wl ~ # ./truss_sample Wait for 10 secs Hello, World! I'm child. PID=11791 Hello, World! PID=11790 root@wl ~ #
※ 첫번째 방법 root@wl ~ # truss ./truss_sample execve("truss_sample", 0x08047D48, 0x08047D50) argc = 1 mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANON, -1, 0) = 0xCFFF0000 resolvepath("/usr/lib/ld.so.1", "/lib/ld.so.1", 1023) = 12 getcwd("/root", 1013) = 0 resolvepath("/root/truss_sample", "/root/truss_sample", 1023) = 28 xstat(2, "/root/truss_sample", 0x08047B28) = 0 open("/var/ld/ld.config", O_RDONLY) Err#2 ENOENT xstat(2, "/usr/local/xml/lib/libc.so.1", 0x08047358) Err#2 ENOENT xstat(2, "/usr/lib/libc.so.1", 0x08047358) = 0 resolvepath("/usr/lib/libc.so.1", "/lib/libc.so.1", 1023) = 14 open("/usr/lib/libc.so.1", O_RDONLY) = 3 mmap(0x00010000, 32768, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_ALIGN, 3, 0) = 0xCFFB0000 mmap(0x00010000, 1212416, PROT_NONE, MAP_PRIVATE|MAP_NORESERVE|MAP_ANON|MAP_ALIGN, -1, 0) = 0xCFE80000 mmap(0xCFE80000, 1101957, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_TEXT, 3, 0) = 0xCFE80000 mmap(0xCFF9E000, 30183, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_INITDATA, 3, 1105920) = 0xCFF9E000 mmap(0xCFFA6000, 4240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANON, -1, 0) = 0xCFFA6000 munmap(0xCFF8E000, 65536) = 0 memcntl(0xCFE80000, 123808, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0 close(3) = 0 sysconfig(_CONFIG_PAGESIZE) = 4096 mmap(0x00010000, 24576, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANON|MAP_ALIGN, -1, 0) = 0xCFF90000 munmap(0xCFFB0000, 32768) = 0 getcontext(0x080478E0) getrlimit(RLIMIT_STACK, 0x080478D8) = 0 getpid() = 20610 [20609] lwp_private(0, 1, 0xCFF92A00) = 0x000001C3 setustack(0xCFF92A60) sysi86(SI86FPSTART, 0xCFFA6740, 0x0000133F, 0x00001F80) = 0x00000001 ioctl(1, TCGETA, 0x08047074) = 0 fstat64(1, 0x08046FE0) = 0 Wait for 10 secs write(1, " W a i t f o r 1 0 ".., 17) = 17 nanosleep(0x08047CF0, 0x08047CF8) (sleeping...) nanosleep(0x08047CF0, 0x08047CF8) = 0 schedctl() = 0xCFFF4000 Hello, World! I'm child. PID=20611 fork1() = 20611 lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF] getpid() = 20610 [20609] Hello, World! PID=20610 write(1, " H e l l o , W o r l d".., 24) = 24 _exit(24) ※ 두번째 방법 root@wl ~ # ./truss_sample & Wait for 10 secs [1] 11893 root@wl ~ # truss -p 11893 nanosleep(0x08047CF0, 0x08047CF8) (sleeping...) nanosleep(0x08047CF0, 0x08047CF8) = 0 schedctl() = 0xCFFF3000 Hello, World! I'm child. PID=11895 fork1() = 11895 lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF] getpid() = 11893 [10585] Hello, World! PID=11893 write(1, " H e l l o , W o r l d".., 24) = 24 _exit(24) [1]+ Exit 24 ./truss_sample ※ 차일드 프로세스까지 트레이스 root@wl ~ # truss -f ./truss_sample ... 23203: fstat64(1, 0x08046F70) = 0 Wait for 10 secs 23203: write(1, " W a i t f o r 1 0 ".., 17) = 17 23203: nanosleep(0x08047C80, 0x08047C88) (sleeping...) 23203: nanosleep(0x08047C80, 0x08047C88) = 0 23203: schedctl() = 0xCFFF4000 23203: fork1() = 23204 23204: fork1() (returning as child ...) = 23203 23204: getpid() = 23204 [23203] 23204: lwp_self() = 1 23204: lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF] 23204: getpid() = 23204 [23203] Hello, World! I'm child. PID=23204 23204: write(1, " H e l l o , W o r l d".., 35) = 35 23204: _exit(35) 23203: lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF] 23203: getpid() = 23203 [23202] Hello, World! PID=23203 23203: write(1, " H e l l o , W o r l d".., 24) = 24 23203: _exit(24)
root@wl ~ # cat exectest.sh
#!/bin/bash
echo "Hello World!"
root@wl ~ # truss -a ./exectest.sh
execve("/usr/bin/bash", 0x08047BB0, 0x08047BBC) argc = 2
argv: /bin/bash ./exectest.sh
sysinfo(SI_MACHINE, "i86pc", 257) = 6
...
root@wl ~ # truss -c ./truss_sample
Wait for 10 secs
Hello, World! I'm child. PID=13729
Hello, World! PID=13673
syscall seconds calls errors
_exit .000 1
write .000 2
open .000 2 1
...
fstat64 .000 1
getcwd .000 1
-------- ------ ----
sys totals: .001 38 2
usr time: .000
elapsed: 10.030
root@wl ~ # truss -d ./truss_sample Base time stamp: 1251782456.0417 [ Tue Sep 1 14:20:56 KST 2009 ] 0.0000 execve("truss_sample", 0x08047D48, 0x08047D50) argc = 1 ... 0.0104 fstat64(1, 0x08046FE0) = 0 Wait for 10 secs 0.0106 write(1, " W a i t f o r 1 0 ".., 17) = 17 nanosleep(0x08047CF0, 0x08047CF8) (sleeping...) 10.0140 nanosleep(0x08047CF0, 0x08047CF8) = 0 10.0142 schedctl() = 0xCFFEE000 Hello, World! I'm child. PID=14311 10.0163 fork1() = 14311 10.0165 lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF] 10.0167 getpid() = 14253 [14252] Hello, World! PID=14253 10.0169 write(1, " H e l l o , W o r l d".., 24) = 24 10.0171 _exit(24)
root@wl ~ # truss -D ./truss_sample
...
0.0000 fstat64(1, 0x08046FE0) = 0
Wait for 10 secs
0.0000 write(1, " W a i t f o r 1 0 ".., 17) = 17
nanosleep(0x08047CF0, 0x08047CF8) (sleeping...)
0.0000 nanosleep(0x08047CF0, 0x08047CF8) = 0
0.0001 schedctl() = 0xCFFF2000
Hello, World! I'm child. PID=22291
0.0002 fork1() = 22291
0.0000 lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF]
0.0000 getpid() = 22290 [22289]
Hello, World! PID=22290
0.0000 write(1, " H e l l o , W o r l d".., 24) = 24
0.0000 _exit(24)
인터럽트 가능한 시스템 콜은 보통 터미널 제어에 사용하는 open(2), read(2)을 의미한다. 이런 시스템 콜은 대부분 문제를 일으키지 않으며, truss로 분석하기 힘들게 만든다.
root@wl ~ # truss -l ./truss_sample
/1: fstat64(1, 0x08046FE0) = 0
Wait for 10 secs
/1: write(1, " W a i t f o r 1 0 ".., 17) = 17
/1: nanosleep(0x08047CF0, 0x08047CF8) (sleeping...)
/1: nanosleep(0x08047CF0, 0x08047CF8) = 0
/1: schedctl() = 0xCFFF2000
Hello, World! I'm child. PID=22551
/1: fork1() = 22551
/1: lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF]
/1: getpid() = 22550 [22549]
Hello, World! PID=22550
/1: write(1, " H e l l o , W o r l d".., 24) = 24
/1: _exit(24)
sys/fault.h를 참조. 기본값은 -m all -m !fltpage-M !allstderr로 출력한다.-r !all-s all-S !all-t all 예) truss -t read,write ./truss_sample-T !all 예) truss -T read,write ./truss_sampleroot@wl ~ # truss -u 'libc:*printf' ./truss_sample ... /1@1: -> libc:printf(0x8050808, 0x0) /1: ioctl(1, TCGETA, 0x08047074) = 0 /1: fstat64(1, 0x08046FE0) = 0 Wait for 10 secs /1: write(1, " W a i t f o r 1 0 ".., 17) = 17 /1@1: <- libc:printf() = 17 /1: nanosleep(0x08047CF0, 0x08047CF8) (sleeping...) /1: nanosleep(0x08047CF0, 0x08047CF8) = 0 /1: schedctl() = 0xCFFF7000 /1: fork1() = 24773 Hello, World! I'm child. PID=24773 /1: lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF] /1: getpid() = 24772 [24771] /1@1: -> libc:printf(0x8050854, 0x60c4) Hello, World! PID=24772 /1: write(1, " H e l l o , W o r l d".., 24) = 24 /1@1: <- libc:printf() = 24 /1: _exit(24) root@wl ~ # truss -u truss_sample -u ld:: -u :: ./truss_sample ... /1@1: -> libc:lfree(0xcff90000, 0x8) /1@1: -> libc:getbucketnum(0x8) /1@1: <- libc:getbucketnum() = 0 /1@1: -> libc:memset(0xcff90000, 0x0, 0x40) /1@1: <- libc:memset() = 0xcff90000 /1@1: -> libc:lmutex_lock(0xcffa3bc0) /1@1: <- libc:lmutex_lock() = 0xcffa3bc0 /1@1: -> libc:lmutex_unlock(0xcffa3bc0) /1@1: <- libc:lmutex_unlock() = 0 /1@1: <- libc:lfree() = 0 /1@1: -> libc:mutex_unlock(0xcffa3e40) /1@1: <- libc:mutex_unlock() = 0 /1@1: <- libc:_exithandle() = 0 /1: _exit(24)
-v !all
root@wl ~ # truss -v fstat64 ./truss_sample
fstat64(1, 0x08046FE0) = 0
d=0x04700000 i=12582922 m=0020620 l=1 u=101 g=7 rdev=0x00600003
at = Sep 1 17:08:12 KST 2009 [ 1251792492 ]
mt = Sep 1 17:08:13 KST 2009 [ 1251792493 ]
ct = Sep 1 13:06:00 KST 2009 [ 1251777960 ]
bsz=8192 blks=1 fs=devfs
Wait for 10 secs
write(1, " W a i t f o r 1 0 ".., 17) = 17
nanosleep(0x08047CF0, 0x08047CF8) (sleeping...)
nanosleep(0x08047CF0, 0x08047CF8) = 0
schedctl() = 0xCFFF9000
Hello, World! I'm child. PID=24008
fork1() = 24008
lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF]
getpid() = 24007 [24006]
Hello, World! PID=24007
write(1, " H e l l o , W o r l d".., 24) = 24
_exit(24)
-w !all
root@wl ~ # truss -w 1 ./truss_sample ... fstat64(1, 0x08046FE0) = 0 Wait for 10 secs write(1, 0xCFFA6B54, 17) = 17 W a i t f o r 1 0 s e c s\n nanosleep(0x08047CF0, 0x08047CF8) (sleeping...) nanosleep(0x08047CF0, 0x08047CF8) = 0 schedctl() = 0xCFFF9000 Hello, World! I'm child. PID=23890 fork1() = 23890 lwp_sigmask(SIG_SETMASK, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF] getpid() = 23889 [23888] Hello, World! PID=23889 write(1, 0xCFFA6B54, 24) = 24 H e l l o , W o r l d ! P I D = 2 3 8 8 9\n _exit(24)
-x !all
root@wl ~ # truss -x lwp_sigmask ./truss_sample
...
fstat64(1, 0x08046FE0) = 0
Wait for 10 secs
write(1, " W a i t f o r 1 0 ".., 17) = 17
nanosleep(0x08047CF0, 0x08047CF8) (sleeping...)
nanosleep(0x08047CF0, 0x08047CF8) = 0
schedctl() = 0xCFFEE000
Hello, World! I'm child. PID=24222
fork1() = 24222
lwp_sigmask(3, 0x00000000, 0x00000000) = 0xFFBFFEFF [0x0000FFFF]
getpid() = 24166 [24165]
Hello, World! PID=24166
write(1, " H e l l o , W o r l d".., 24) = 24
_exit(24)
-t 옵션이다. 어플리케이션이 어떤 시스템 콜을 봐야하는지를 알려면, 약간의 시스템 프로그래밍적 지식이 필요하다고 본다. 간단한 프로그래밍 지식과 시스템 콜과 관련된 문서를 읽어보는 것을 추천한다.root@wl ~ # apptrace ./truss_sample -> truss_sample -> libc.so.1:int atexit(int (*)() = 0xcffd5ed8) <- truss_sample -> libc.so.1:atexit() -> truss_sample -> libc.so.1:int atexit(int (*)() = 0x80507e8) <- truss_sample -> libc.so.1:atexit() -> truss_sample -> libc.so.1:void __fpstart(void) <- truss_sample -> libc.so.1:__fpstart() = 0xcfd266fc -> truss_sample -> libc.so.1:int printf(const char * = 0x8050808 "Wait for 10 secs", void * = 0x0, ...) Wait for 10 secs <- truss_sample -> libc.so.1:printf() = 0x11 -> truss_sample -> libc.so.1:unsigned sleep(unsigned = 0xa) <- truss_sample -> libc.so.1:sleep() -> truss_sample -> libc.so.1:pid_t fork(void) <- truss_sample -> libc.so.1:fork() -> truss_sample -> libc.so.1:getpid(0x0, 0x8047ce0, 0x80506bd) ** NR -> truss_sample -> libc.so.1:int printf(const char * = 0x8050830 "Hello, World! I'm child. PID=%d", void * = 0x51a5, ...) Hello, World! I'm child. PID=20901 <- truss_sample -> libc.so.1:printf() = 0x23 -> truss_sample -> libc.so.1:exit(0x23, 0x8047ce0, 0x805065f) ** NR <- truss_sample -> libc.so.1:fork() = 0x51a5 -> truss_sample -> libc.so.1:getpid(0x51a5, 0x8047ce0, 0x80506bd) ** NR -> truss_sample -> libc.so.1:int printf(const char * = 0x8050854 "Hello, World! PID=%d", void * = 0x51a4, ...) Hello, World! PID=20900 <- truss_sample -> libc.so.1:printf() = 0x18 -> truss_sample -> libc.so.1:exit(0x18, 0x8047ce0, 0x805065f) ** NR root@wl ~ #
truss_tnf.c | (917 바이트) |
/*
* SEE tracing(3TNF)
*
* To compile in all probes (for development):
* cc -DTNF_DEBUG ...
*
* To compile in only production probes (for release):
* cc ...
*
* To compile in no probes at all:
* cc -DNPROBE ...
*/
#include <tnf/probe.h>
void work(long, char *);
enum work_request_type { READ, WRITE, ERASE, UPDATE };
static char *work_request_name[] = {"read", "write", "erase", "update"};
void main()
{
long i;
for (i = READ; i <= UPDATE; i++) {
work(i, work_request_name[i]);
}
}
void work(long request_type, char *request_name)
{
static long q_length;
TNF_PROBE_2_DEBUG(work_start, "work", "XYZ%debug 'in function work'", tnf_long, request_type_arg, request_type, tnf_string, request_name_arg, request_name);
q_length++;
TNF_PROBE_1(work_queue, "work queue", "XYZ%work_load heavy", tnf_long, queue_length, q_length);
TNF_PROBE_0_DEBUG(work_end, "work", "");
}
windy@wl ~ $ cc -DTNF_DEBUG -o tnf truss_tnf.c
windy@wl ~ $ prex ./tnf
대상 프로세스가 정지되었습니다
대상을 계속하려면 "continue"을 입력하고 도움말을 보려면 "help"를 입력하십시오 ...
/usr/bin/i86/prex> list probes $all
name=work_start enable=off trace=on file=tnf.c line=15 funcs=<no value>
name=work_queue enable=off trace=on file=tnf.c line=17 funcs=<no value>
name=work_end enable=off trace=on file=tnf.c line=18 funcs=<no value>
/usr/bin/i86/prex> enable $all
/usr/bin/i86/prex> continue
/usr/bin/i86/prex: 대상 프로세스가 종료되었습니다
windy@wl ~ $ tnfdump /tmp/trace-*
probe tnf_name: "work_start" tnf_string: "keys work;file tnf.c;line 15;debug;XYZ%debug 'in function work'"
probe tnf_name: "work_queue" tnf_string: "keys work queue;file tnf.c;line 17;XYZ%work_load heavy"
probe tnf_name: "work_end" tnf_string: "keys work;file tnf.c;line 18;debug;"
---------------- ---------------- ----- ----- ---------- --- ------------------------- ------------------------
Elapsed (ms) Delta (ms) PID LWPID TID CPU Probe Name Data / Description . . .
---------------- ---------------- ----- ----- ---------- --- ------------------------- ------------------------
0.000000 0.000000 3743 1 1 - work_start request_type_arg: 0 request_name_arg: "read"
0.016040 0.016040 3743 1 1 - work_queue queue_length: 1
0.018334 0.002294 3743 1 1 - work_end
0.018759 0.000425 3743 1 1 - work_start request_type_arg: 1 request_name_arg: "write"
0.019364 0.000605 3743 1 1 - work_queue queue_length: 2
0.019739 0.000375 3743 1 1 - work_end
0.020110 0.000371 3743 1 1 - work_start request_type_arg: 2 request_name_arg: "erase"
0.020648 0.000538 3743 1 1 - work_queue queue_length: 3
0.021007 0.000359 3743 1 1 - work_end
0.021376 0.000369 3743 1 1 - work_start request_type_arg: 3 request_name_arg: "update"
0.021951 0.000575 3743 1 1 - work_queue queue_length: 4
0.022308 0.000357 3743 1 1 - work_end
root@wl ~ # prex -k
도움말을 보려면 "help"를 입력하십시오...
prex> buffer alloc 2m
2097152 바이트 크기의 버퍼가 할당되었습니다
prex> list probes $all
...
prex> list probes syscall
name=syscall_end enable=off trace=off file=../../intel/ia32/os/syscall.c line=717 funcs=<no value>
name=syscall_start enable=off trace=off file=../../intel/ia32/os/syscall.c line=389 funcs=<no value>
prex> enable syscall
prex> trace syscall
prex> ktrace on
여기서 다른 콘솔을 열어 트레이스할 프로그램들을 실행한다. 이후 아래와 같이 진행
prex> ktrace off
prex> quit
root@wl ~ # tnfxtract /tmp/ktrace 1)
root@wl ~ # tnfdump /tmp/ktrace
root@wl ~ # prex -k
도움말을 보려면 "help"를 입력하십시오...
prex> untrace $all
prex> disable $all
prex> list probes syscall
name=syscall_end enable=off trace=off file=../../intel/ia32/os/syscall.c line=717 funcs=<no value>
name=syscall_start enable=off trace=off file=../../intel/ia32/os/syscall.c line=389 funcs=<no value>
prex> buffer dealloc
버퍼 할당이 해제되었습니다
prex> quit
1) /tmp/ktrace파일과 /tmp/trace-PID파일이 생성된다.
truss_compare.c | (893 바이트) |
// Based On http://milek.blogspot.com/2009/04/truss1m-vs-dtrace1m.html
#include <thread.h>
#include <stdlib.h>
#include <pthread.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <stdio.h>
void *thread_func(void *arg)
{
int *N=arg;
int i;
struct stat buf;
for (i=0; i<*N; i++)
stat("/tmp", &buf);
return(0);
}
int main(int argc, char **argv)
{
int N, iter;
int i;
int rc;
pthread_t tid[255];
if (argc != 3)
{
printf("%s number_of_threads number_of_iterations_per_thread\n", argv[0]);
exit(1);
}
N = atoi(argv[1]);
iter = atoi(argv[2]);
for (i=0; i<N; i++)
{
if (rc = pthread_create(&tid[i], NULL, thread_func, &iter))
printf("Thread #%d creation failed [%d]\n", i, rc);
}
/* wait for all threads to complete */
for (i=0; i<N; i++)
pthread_join(tid[i], NULL);
exit(0);
}
root@wl ~/t # cc -fast truss_compare.c -o threads-2
root@wl ~/t # ptime truss -t xstat -c ./threads-2 10 100000
syscall seconds calls errors
xstat 6.108 1000000
stat64 .000 7 5
-------- ------ ----
sys totals: 6.108 1000007 5
usr time: .925
elapsed: 23.050
real 23.056
user 8.380
sys 22.410
root@wl ~/t # ptime dtrace -n 'syscall::stat:entry{@=count();}' -c 'ptime ./threads-2 10 100000'
dtrace: description 'syscall::stat:entry' matched 1 probe
real 1.267
user 0.225
sys 2.278
dtrace: pid 15368 has exited
46
real 1.673
user 0.280
sys 0.119
root@wl ~/t # ptime ./threads-2 10 100000
real 1.260
user 0.224
sys 2.277
root@wl ~/t #
| RSS ATOM XHTML 5 CSS3 |
Copyright © 2004-2025 Jo HoSeok. All rights reserved. |