feat: use caching to speed up installation

缓存deb包的data.tar, 避免反复解压文件, 加快安装速度
linuxdeepin · Nov 13, 2024 · bb85320 · bb85320
1 parent d6f519d
commit bb85320
Showing 1 changed file with 65 additions and 56 deletions.
diff --git a/misc/libexec/linglong/builder/helper/install_dep b/misc/libexec/linglong/builder/helper/install_dep
@@ -1,5 +1,8 @@
 #!/bin/bash
 set -e
+project_dir=$PWD
+cache_dir=${LINGLONG_FETCH_CACHE:-$PWD/linglong/cache}
+mkdir -p "$cache_dir"
 # 文件名 deb-source.bash
 # 包含要解压的deb目录
 deb_dir=$(realpath "$1")
@@ -26,73 +29,79 @@ mkdir /tmp/deb-source-file || true
 
 # 如果base和runtime已安装则跳过，旧版本base没有/packages.list文件就使用/var/lib/dpkg/status
 grep 'Package: ' /var/lib/dpkg/status > "$exclude_list_file" || true
-cat /packages.list /runtime/packages.list "$PREFIX/packages.list" >> "$exclude_list_file" || true
-# 在旧的base里面这些包需要强制安装，因为base中没有他们的dev包，如果dev包被安装到/opt目录，而lib包在/usr 会有问题
-echo "libarchive13,libasan5,libasm1,libbabeltrace1,libcairo-script-interpreter2,libcc1-0,libcurl4,libdpkg-perl,libdw1,libevent-2.1-6,libgdbm-compat4,libgdbm6,libgirepository-1.0-1,libgles1,libgles2,libglib2.0-data,libgmpxx4ldbl,libgnutls-dane0,libgnutls-openssl27,libgnutlsxx28,libharfbuzz-gobject0,libharfbuzz-icu0,libipt2,libisl19,libitm1,libjsoncpp1,libldap-2.4-2,libldap-common,liblsan0,liblzo2-2,libmpc3,libmpdec2,libmpfr6,libmpx2,libncurses6,libnghttp2-14,libpcrecpp0v5,libperl5.28,libpopt0,libprocps7,libpython3-stdlib,libpython3.7,libpython3.7-minimal,libpython3.7-stdlib,libquadmath0,libreadline7,librhash0,librtmp1,libsasl2-2,libsasl2-modules-db,libssh2-1,libtiffxx5,libtsan0,libubsan1,libunbound8,libuv1" | tr ',' '\n' >> "$include_list_file"
+# 跳过base和runtime已安装的包，也可使用install_dep_skip.list文件控制跳过哪些包
+cat /packages.list /runtime/packages.list "$PREFIX/packages.list" "$project_dir/install_dep_skip.list" >> "$exclude_list_file" || true
 
 # 遍历文件列表
-while IFS= read -r file
+while IFS= read -r deb_file
 do
     # 输出deb名，但不换行，便于在包名后面加skip
-    echo -n "$file"
+    echo -n "$deb_file"
     # 提取control文件
-    control_file=$(ar -t "$file" | grep control.tar)
-    ar -x "$file" "$control_file"
+    control_file=$(ar -t "$deb_file" | grep control.tar)
+    ar -x "$deb_file" "$control_file"
     # 获取包名
     pkg=$(tar -xf "$control_file" ./control -O | grep '^Package:' | awk '{print $2}')
     rm "$control_file"
-    # 如果在base和runtime中已安装，并且不包含在include列表则跳过安装
+    # 如果在base和runtime中已安装，并且不包含在include（强制安装）列表则跳过安装，否则安装到$PREFIX目录
     if grep -q "^Package: $pkg$" "$exclude_list_file" && ! grep -q "^$pkg$" "$include_list_file"; then
         echo " skip"
-        echo "$file" >> /tmp/deb-source-file/skip.list
-    else
-    # 否则安装到$PREFIX目录
-        # 换行
-        echo ""
-        # 查找data.tar文件，文件会因为压缩格式不同，有不同的后缀，例如data.tar.xz、data.tar.gz
-        data_file=$(ar -t "$file" | grep data.tar)
-        # 提取data.tar文件
-        ar -x "$file" "$data_file"
-        # 解压data.tar文件到输出目录
-        mkdir "$data_list_dir"
-        tar -xvf "$data_file" -C "$data_list_dir" >> "/tmp/deb-source-file/$(basename "$file").list"
-        rm "$data_file"
-
-        # 清理不需要复制的目录
-        rm -r "${data_list_dir:?}/usr/share/applications"* 2>/dev/null || true
-        # 修改pc文件的prefix
-        sed -i "s#/usr#$PREFIX#g" "$data_list_dir"/usr/lib/"$TRIPLET"/pkgconfig/*.pc 2>/dev/null|| true
-        sed -i "s#/usr#$PREFIX#g" "$data_list_dir"/usr/share/pkgconfig/*.pc 2>/dev/null || true
-        # 修改指向/lib的绝对路径的软链接
-        find "$data_list_dir" -type l | while IFS= read -r file; do
-            linkTarget=$(readlink "$file") 
-            # 如果指向的路径以/lib开头，并且文件不存在，则添加 /runtime 前缀
-            # 部分 dev 包会创建 so 文件的绝对链接指向 /lib 目录下
-            if echo "$linkTarget" | grep -q ^/lib && ! [ -f "$linkTarget" ]; then
-                ln -sf "$target$linkTarget" "$file"
-                echo "    FIX LINK" "$linkTarget" "=>" "$target$linkTarget"
-            fi
-        done
-        # 修复动态库的RUNPATH
-        find "$data_list_dir" -type f -exec file {} \; | grep 'shared object' | awk -F: '{print $1}' | while IFS= read -r file; do
-            runpath=$(readelf -d "$file" | grep RUNPATH |  awk '{print $NF}')
-            # 如果RUNPATH使用绝对路径，则添加/runtime前缀
-            if echo "$runpath" | grep -q '^\[/'; then
-                runpath=${runpath#[}
-                runpath=${runpath%]}
-                newRunpath=${runpath//usr\/lib/runtime\/lib}
-                newRunpath=${newRunpath//usr/runtime}
-                patchelf --set-rpath "$newRunpath" "$file"
-                echo "    FIX RUNPATH" "$file" "$runpath" "=>" "$newRunpath"
-            fi
-        done
-        # 复制/lib,/bin,/usr目录
-        cp -rP "$data_list_dir/lib" "$target" 2>/dev/null|| true
-        cp -rP "$data_list_dir/bin" "$target" 2>/dev/null|| true
-        cp -rP "$data_list_dir"/usr/* "$target" || true
-        rm -r "$data_list_dir"
-
+        echo "$deb_file" >> /tmp/deb-source-file/skip.list
+        continue
     fi;
+    # 记录到 packages.json
+    echo "Package: $pkg" >> "$PREFIX/packages.json"
+    # 换行
+    echo ""
+    # 缓存解压后的data.tar文件，便于在下次使用时，加快安装速度
+    deb_sha=$(sha256sum "$deb_file" | awk '{print $1}')
+    data_cache="$cache_dir/install_dep_$deb_sha"
+    if [ ! -e "$data_cache" ]; then
+        # 查找data.tar文件，文件会因为压缩格式不同，有不同的后缀，例如data.tar.xz、data.tar.gz
+        data_file=$(ar -t "$deb_file" | grep data.tar)
+        case "$data_file" in
+            *.xz ) ar -p "$deb_file" "$data_file" | unxz > "$data_cache" ;;
+            *.gz ) ar -p "$deb_file" "$data_file" | gunzip > "$data_cache" ;;
+            *.zst ) ar -p "$deb_file" "$data_file" | unzstd > "$data_cache" ;;
+            * ) echo "unknown file type"; exit 1;;
+        esac
+    fi
+    # 解压data.tar文件到输出目录
+    mkdir "$data_list_dir"
+    tar -xvf "$data_cache" -C "$data_list_dir" >> "/tmp/deb-source-file/$(basename "$deb_file").list"
+    # 清理不需要复制的目录
+    rm -r "${data_list_dir:?}/usr/share/applications"* 2>/dev/null || true
+    # 修改pc文件的prefix
+    sed -i "s#/usr#$PREFIX#g" "$data_list_dir"/usr/lib/"$TRIPLET"/pkgconfig/*.pc 2>/dev/null|| true
+    sed -i "s#/usr#$PREFIX#g" "$data_list_dir"/usr/share/pkgconfig/*.pc 2>/dev/null || true
+    # 修改指向/lib的绝对路径的软链接
+    find "$data_list_dir" -type l | while IFS= read -r file; do
+        linkTarget=$(readlink "$file") 
+        # 如果指向的路径以/lib开头，并且文件不存在，则添加 /runtime 前缀
+        # 部分 dev 包会创建 so 文件的绝对链接指向 /lib 目录下
+        if echo "$linkTarget" | grep -q ^/lib && ! [ -f "$linkTarget" ]; then
+            ln -sf "$target$linkTarget" "$file"
+            echo "    FIX LINK" "$linkTarget" "=>" "$target$linkTarget"
+        fi
+    done
+    # 修复动态库的RUNPATH
+    find "$data_list_dir" -type f -exec file {} \; | grep 'shared object' | awk -F: '{print $1}' | while IFS= read -r file; do
+        runpath=$(readelf -d "$file" | grep RUNPATH |  awk '{print $NF}')
+        # 如果RUNPATH使用绝对路径，则添加/runtime前缀
+        if echo "$runpath" | grep -q '^\[/'; then
+            runpath=${runpath#[}
+            runpath=${runpath%]}
+            newRunpath=${runpath//usr\/lib/runtime\/lib}
+            newRunpath=${newRunpath//usr/runtime}
+            patchelf --set-rpath "$newRunpath" "$file"
+            echo "    FIX RUNPATH" "$file" "$runpath" "=>" "$newRunpath"
+        fi
+    done
+    # 复制/lib,/bin,/usr目录
+    cp -rP "$data_list_dir/lib" "$target" 2>/dev/null|| true
+    cp -rP "$data_list_dir/bin" "$target" 2>/dev/null|| true
+    cp -rP "$data_list_dir"/usr/* "$target" || true
+    rm -r "$data_list_dir"
 done < "$deb_list_file"