From 4d56383025f2cbd00dc6296161e31a896624ab75 Mon Sep 17 00:00:00 2001 From: "fuchen.ljl" Date: Wed, 6 Dec 2023 20:23:56 +0800 Subject: Long distance memory overflow issue Problem: The memory will slowly increase with the drawing until restarting. Observation: GC analysis shows that no occupation has occurred, so it is suspected to be a problem with the underlying allocator. Reason: Under Linux, glibc is used to allocate memory. glibc uses brk and mmap to allocate memory, and the memory allocated by brk cannot be released until the high-address memory is released. That is to say, if you apply for two pieces of memory A and B through brk, it is impossible to release A before B is released, and it is still occupied by the process. Check the suspected "memory leak" through TOP. So I replaced TCMalloc, but found that libtcmalloc_minimal could not find ptthread_Key_Create. After analysis, it was found that pthread was not entered during compilation. --- webui.sh | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) (limited to 'webui.sh') diff --git a/webui.sh b/webui.sh index 3d0f87ee..081624c4 100755 --- a/webui.sh +++ b/webui.sh @@ -222,13 +222,29 @@ fi # Try using TCMalloc on Linux prepare_tcmalloc() { if [[ "${OSTYPE}" == "linux"* ]] && [[ -z "${NO_TCMALLOC}" ]] && [[ -z "${LD_PRELOAD}" ]]; then - TCMALLOC="$(PATH=/usr/sbin:$PATH ldconfig -p | grep -Po "libtcmalloc(_minimal|)\.so\.\d" | head -n 1)" - if [[ ! -z "${TCMALLOC}" ]]; then - echo "Using TCMalloc: ${TCMALLOC}" - export LD_PRELOAD="${TCMALLOC}" - else - printf "\e[1m\e[31mCannot locate TCMalloc (improves CPU memory usage)\e[0m\n" - fi + # Define Tcmalloc Libs arrays + TCMALLOC_LIBS=("libtcmalloc(_minimal|)\.so\.\d" "libtcmalloc\.so\.\d") + + # Traversal array + for lib in "${TCMALLOC_LIBS[@]}" + do + #Determine which type of tcmalloc library the library supports + TCMALLOC="$(PATH=/usr/sbin:$PATH ldconfig -p | grep -P $lib | head -n 1)" + TC_INFO=(${TCMALLOC//=>/}) + if [[ ! -z "${TC_INFO}" ]]; then + echo "Using TCMalloc: ${TC_INFO}" + #Determine if the library is linked to libptthread and resolve undefined symbol: ptthread_Key_Create + if ldd ${TC_INFO[2]} | grep -q 'libpthread'; then + echo "$TC_INFO is linked with libpthread,execute LD_PRELOAD=${TC_INFO}" + export LD_PRELOAD="${TC_INFO}" + break + else + echo "$TC_INFO is not linked with libpthreadand will trigger undefined symbol: ptthread_Key_Create error" + fi + else + printf "\e[1m\e[31mCannot locate TCMalloc (improves CPU memory usage)\e[0m\n" + fi + done fi } -- cgit v1.2.3 From bda86f0fd9653657c146f7c1128f92771d16ad4e Mon Sep 17 00:00:00 2001 From: Hina <102651522+HinaHyugaHime@users.noreply.github.com> Date: Tue, 12 Dec 2023 19:39:14 -0600 Subject: Update webui.sh --- webui.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'webui.sh') diff --git a/webui.sh b/webui.sh index 3d0f87ee..046ecf9d 100755 --- a/webui.sh +++ b/webui.sh @@ -131,7 +131,7 @@ case "$gpu_info" in if [[ $(bc <<< "$pyv <= 3.10") -eq 1 ]] then # Navi users will still use torch 1.13 because 2.0 does not seem to work. - export TORCH_COMMAND="pip install torch==1.13.1+rocm5.2 torchvision==0.14.1+rocm5.2 --index-url https://download.pytorch.org/whl/rocm5.2" + export TORCH_COMMAND="pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm5.6" else printf "\e[1m\e[31mERROR: RX 5000 series GPUs must be using at max python 3.10, aborting...\e[0m" exit 1 @@ -141,9 +141,8 @@ case "$gpu_info" in *"Navi 2"*) export HSA_OVERRIDE_GFX_VERSION=10.3.0 ;; *"Navi 3"*) [[ -z "${TORCH_COMMAND}" ]] && \ - export TORCH_COMMAND="pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm5.6" - # Navi 3 needs at least 5.5 which is only on the nightly chain, previous versions are no longer online (torch==2.1.0.dev-20230614+rocm5.5 torchvision==0.16.0.dev-20230614+rocm5.5 torchaudio==2.1.0.dev-20230614+rocm5.5) - # so switch to nightly rocm5.6 without explicit versions this time + export TORCH_COMMAND="pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm5.7" + ;; *"Renoir"*) export HSA_OVERRIDE_GFX_VERSION=9.0.0 printf "\n%s\n" "${delimiter}" -- cgit v1.2.3 From ec124607f47371a6cfd61a795f86a7f1cbd44651 Mon Sep 17 00:00:00 2001 From: wangshuai09 <391746016@qq.com> Date: Sat, 27 Jan 2024 17:21:32 +0800 Subject: Add NPU Support --- webui.sh | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'webui.sh') diff --git a/webui.sh b/webui.sh index cff43327..3f6e87fd 100755 --- a/webui.sh +++ b/webui.sh @@ -159,6 +159,10 @@ then if echo "$gpu_info" | grep -q "AMD" && [[ -z "${TORCH_COMMAND}" ]] then export TORCH_COMMAND="pip install torch==2.0.1+rocm5.4.2 torchvision==0.15.2+rocm5.4.2 --index-url https://download.pytorch.org/whl/rocm5.4.2" + elif echo "$gpu_info" | grep -q "Huawei" && [[ -z "${TORCH_COMMAND}" ]] + then + export TORCH_COMMAND="pip install torch==2.1.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu; pip install torch_npu" + fi fi -- cgit v1.2.3 From 2ba0277b52d352966046cb7b0c5c047cd8a7cdd4 Mon Sep 17 00:00:00 2001 From: analysisjp Date: Sat, 10 Feb 2024 10:09:19 +0900 Subject: fix: prepare_tcmalloc (Fixed memory leak issue in Ubuntu 22.04 or modern linux environment) --- webui.sh | 54 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-) (limited to 'webui.sh') diff --git a/webui.sh b/webui.sh index 25b94906..794cfb8a 100755 --- a/webui.sh +++ b/webui.sh @@ -226,30 +226,48 @@ fi # Try using TCMalloc on Linux prepare_tcmalloc() { if [[ "${OSTYPE}" == "linux"* ]] && [[ -z "${NO_TCMALLOC}" ]] && [[ -z "${LD_PRELOAD}" ]]; then + # check glibc version + LIBC_LIB="$(PATH=/usr/sbin:$PATH ldconfig -p | grep -P "libc.so.6" | head -n 1)" + LIBC_INFO=$(echo ${LIBC_LIB} | awk '{print $NF}') + LIBC_VER=$(echo $(${LIBC_INFO} | awk 'NR==1 {print $NF}') | grep -oP '\d+\.\d+') + echo "glibc version is $LIBC_VER" + libc_vernum=$(expr $LIBC_VER) + # Since 2.34 libpthread is integrated into libc.so + libc_v234=2.34 # Define Tcmalloc Libs arrays TCMALLOC_LIBS=("libtcmalloc(_minimal|)\.so\.\d" "libtcmalloc\.so\.\d") - # Traversal array for lib in "${TCMALLOC_LIBS[@]}" do - #Determine which type of tcmalloc library the library supports - TCMALLOC="$(PATH=/usr/sbin:$PATH ldconfig -p | grep -P $lib | head -n 1)" - TC_INFO=(${TCMALLOC//=>/}) - if [[ ! -z "${TC_INFO}" ]]; then - echo "Using TCMalloc: ${TC_INFO}" - #Determine if the library is linked to libptthread and resolve undefined symbol: ptthread_Key_Create - if ldd ${TC_INFO[2]} | grep -q 'libpthread'; then - echo "$TC_INFO is linked with libpthread,execute LD_PRELOAD=${TC_INFO}" - export LD_PRELOAD="${TC_INFO}" - break - else - echo "$TC_INFO is not linked with libpthreadand will trigger undefined symbol: ptthread_Key_Create error" - fi - else - printf "\e[1m\e[31mCannot locate TCMalloc (improves CPU memory usage)\e[0m\n" - fi + # Determine which type of tcmalloc library the library supports + TCMALLOC="$(PATH=/usr/sbin:$PATH ldconfig -p | grep -P $lib | head -n 1)" + TC_INFO=(${TCMALLOC//=>/}) + if [[ ! -z "${TC_INFO}" ]]; then + echo "Check TCMalloc: ${TC_INFO}" + # Determine if the library is linked to libptthread and resolve undefined symbol: ptthread_key_create + if [ $(echo "$libc_vernum < $libc_v234" | bc) -eq 1 ]; then + # glibc < 2.33 pthread_key_create into libpthead.so. check linking libpthread.so... + if ldd ${TC_INFO[2]} | grep -q 'libpthread'; then + echo "$TC_INFO is linked with libpthread,execute LD_PRELOAD=${TC_INFO[2]}" + # set fullpath LD_PRELOAD (To be on the safe side) + export LD_PRELOAD="${TC_INFO[2]}" + break + else + echo "$TC_INFO is not linked with libpthread will trigger undefined symbol: pthread_Key_create error" + fi + else + # Version 2.34 of libc.so (glibc) includes the pthead library IN GLIBC. (USE ubuntu 22.04 and modern linux system and WSL) + # libc.so(glibc) is linked with a library that works in ALMOST ALL Linux userlands. SO NO CHECK! + echo "$TC_INFO is linked with libc.so,execute LD_PRELOAD=${TC_INFO[2]}" + # set fullpath LD_PRELOAD (To be on the safe side) + export LD_PRELOAD="${TC_INFO[2]}" + break + fi + fi done - + if [[ -z "${LD_PRELOAD}" ]]; then + printf "\e[1m\e[31mCannot locate TCMalloc. Do you have tcmalloc or gperftools installed on your system? (improves CPU memory usage)\e[0m\n" + fi fi } -- cgit v1.2.3 From 69f9564a6db8ced1b5fbfc7f20c644abd54fca11 Mon Sep 17 00:00:00 2001 From: analysisjp Date: Tue, 13 Feb 2024 21:49:23 +0900 Subject: fixed webui.sh issue that occurred in WSL environment (fix: #14883) --- webui.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'webui.sh') diff --git a/webui.sh b/webui.sh index 794cfb8a..f116376f 100755 --- a/webui.sh +++ b/webui.sh @@ -227,9 +227,7 @@ fi prepare_tcmalloc() { if [[ "${OSTYPE}" == "linux"* ]] && [[ -z "${NO_TCMALLOC}" ]] && [[ -z "${LD_PRELOAD}" ]]; then # check glibc version - LIBC_LIB="$(PATH=/usr/sbin:$PATH ldconfig -p | grep -P "libc.so.6" | head -n 1)" - LIBC_INFO=$(echo ${LIBC_LIB} | awk '{print $NF}') - LIBC_VER=$(echo $(${LIBC_INFO} | awk 'NR==1 {print $NF}') | grep -oP '\d+\.\d+') + LIBC_VER=$(echo $(ldd --version | awk 'NR==1 {print $NF}') | grep -oP '\d+\.\d+') echo "glibc version is $LIBC_VER" libc_vernum=$(expr $LIBC_VER) # Since 2.34 libpthread is integrated into libc.so @@ -244,9 +242,9 @@ prepare_tcmalloc() { TC_INFO=(${TCMALLOC//=>/}) if [[ ! -z "${TC_INFO}" ]]; then echo "Check TCMalloc: ${TC_INFO}" - # Determine if the library is linked to libptthread and resolve undefined symbol: ptthread_key_create + # Determine if the library is linked to libpthread and resolve undefined symbol: pthread_key_create if [ $(echo "$libc_vernum < $libc_v234" | bc) -eq 1 ]; then - # glibc < 2.33 pthread_key_create into libpthead.so. check linking libpthread.so... + # glibc < 2.34 pthread_key_create into libpthread.so. check linking libpthread.so... if ldd ${TC_INFO[2]} | grep -q 'libpthread'; then echo "$TC_INFO is linked with libpthread,execute LD_PRELOAD=${TC_INFO[2]}" # set fullpath LD_PRELOAD (To be on the safe side) @@ -256,7 +254,7 @@ prepare_tcmalloc() { echo "$TC_INFO is not linked with libpthread will trigger undefined symbol: pthread_Key_create error" fi else - # Version 2.34 of libc.so (glibc) includes the pthead library IN GLIBC. (USE ubuntu 22.04 and modern linux system and WSL) + # Version 2.34 of libc.so (glibc) includes the pthread library IN GLIBC. (USE ubuntu 22.04 and modern linux system and WSL) # libc.so(glibc) is linked with a library that works in ALMOST ALL Linux userlands. SO NO CHECK! echo "$TC_INFO is linked with libc.so,execute LD_PRELOAD=${TC_INFO[2]}" # set fullpath LD_PRELOAD (To be on the safe side) @@ -266,7 +264,7 @@ prepare_tcmalloc() { fi done if [[ -z "${LD_PRELOAD}" ]]; then - printf "\e[1m\e[31mCannot locate TCMalloc. Do you have tcmalloc or gperftools installed on your system? (improves CPU memory usage)\e[0m\n" + printf "\e[1m\e[31mCannot locate TCMalloc. Do you have tcmalloc or google-perftool installed on your system? (improves CPU memory usage)\e[0m\n" fi fi } -- cgit v1.2.3