Merge pull request #5855 from BOINC/dpa_docker_wrapper

Add docker_wrapper and associated client/server changes

client/Makefile.linux

@@ -140,7 +140,7 @@ clean:
 LIBS = ../lib/boinc.a \
     -L /usr/local/lib/ \
     -lpthread \
-    -lX11 -lXss \
+    -lX11 \
     -lcurl -lssl -lcrypto \
     -lz -ldl
 

client/app_test.cpp

@@ -15,37 +15,68 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
-// A framework that lets you run jobs under a BOINC client
-// without a project, and without fake XML files
-// This lets you debug client/app interactions.
+// This framework lets you run jobs under a BOINC client without a project.
+// This lets you debug client/app interactions like suspend/resume.
+// The properties of the app and its files
+// are described procedurally (in this file)
+// rather than with XML files.
 //
 // To use this framework:
 // - edit this file to describe your application:
 //      input/output files, attributes, etc.
-//      NOTE: currently it's set up for an app that uses the WSL wrapper
-//
+//      It currently has several test cases, selected with #ifdef
 // - build the BOINC client with these changes
 // - make a BOINC data directory, say 'test'
 //      (or you can use an existing BOINC data directory,
-//      in which case the client will also run jobs that are there)
+//      in which case the client will also run existing jobs)
 // - make a directory test/slots/app_test
-//      The client will run the test job there.
+//      The client will run your test job there.
 //      Clean it out between runs.
 // - make a dir test/projects/app_test
 // - In the project directory, put:
 //      - the executable file
 //      - the input file(s) with physical names
-// - run boinc (in the data directory if you created one)
-//      when the job is done, the client won't clean out the slot dir.
-//      You can examine the contents of the slot dir,
-//      and examine the output files in the project dir.
+//      NOTE: slots/app_test and projects/app_test can be symbolic links
+//      in case you have multiple test cases
+// - run boinc in the data directory, e.g. test/
+//      The client will copy files and create link files
+//      as needed in the slot dir,
+//      and create init_data.xml there.
+//      When the job is done, the client won't clean out the slot dir.
+//      You can examine the contents of the slot and project dir,
 //      Clean out the slot dir between tests.
 
 #include "client_state.h"
 
-// set to 0 to enable app test
+// define exactly one
+
+//#define APP_NONE
+//#define APP_WSL_WRAPPER
+//      type    physical            logical             copy?
+//      app     wsl_wrapper.exe     wsl_wrapper.exe
+//      app     worker              worker
+//      app     main                main                yes
+//      input   infile              in
+//      output  outfile             out
+#define APP_DOCKER_WRAPPER_COPY
+//      type    physical            logical             copy?
+//      app     worker              worker              yes
+//      app     job_copy.toml       job_copy.toml       yes
+//      app     Dockerfile_copy     Dockerfile_copy     yes
+//      app     docker_wrapper      docker_wrapper
+//      input   infile              in                  yes
+//      output  outfile             out                 yes
+//#define APP_DOCKER_WRAPPER_MOUNT
+//      type    physical            logical             copy?
+//      app     worker              worker
+//      app     job_mount.toml      job_mount.toml      yes
+//      app     Dockerfile_mount    Dockerfile_mount    yes
+//      app     main.sh             main.sh             yes
+//      app     docker_wrapper      docker_wrapper
+//      input   infile              in
+//      output  outfile             out
 
-#if 1
+#ifdef APP_NONE
 void CLIENT_STATE::app_test_init() {}
 #else
 
@@ -83,6 +114,8 @@ static APP* make_app(PROJECT* proj) {
 #define OUTPUT_FILE 1
 #define MAIN_PROG   2
 
+// if log_name is NULL, logical name is physical name
+//
 static FILE_REF* make_file(
     PROJECT *proj, const char* phys_name, const char* log_name,
     int ftype, bool copy_file
@@ -163,6 +196,10 @@ void CLIENT_STATE::app_test_init() {
 #endif
 
     APP_VERSION *av = make_app_version(app);
+
+////////////// APP VERSION FILES /////////////////
+
+#ifdef APP_WSL_WRAPPER
     av->app_files.push_back(
         *make_file(app->project, "wsl_wrapper.exe", NULL, MAIN_PROG, false)
     );
@@ -172,6 +209,21 @@ void CLIENT_STATE::app_test_init() {
     av->app_files.push_back(
         *make_file(app->project, "worker", NULL, INPUT_FILE, false)
     );
+#endif
+#ifdef APP_DOCKER_WRAPPER_COPY
+    av->app_files.push_back(
+        *make_file(app->project, "docker_wrapper.exe", NULL, MAIN_PROG, false)
+    );
+    av->app_files.push_back(
+        *make_file(app->project, "worker", NULL, INPUT_FILE, true)
+    );
+    av->app_files.push_back(
+        *make_file(app->project, "job_copy.toml", "job.toml", INPUT_FILE, true)
+    );
+    av->app_files.push_back(
+        *make_file(app->project, "Dockerfile_copy", "Dockerfile", INPUT_FILE, true)
+    );
+#endif
 
     // can put other stuff here like
 #if 0
@@ -181,19 +233,35 @@ void CLIENT_STATE::app_test_init() {
 #endif
 
     WORKUNIT *wu = make_workunit(av);
-#if 1
+
+////////////// INPUT FILES /////////////////
+
+#ifdef APP_WSL_WRAPPER
     //wu->command_line = "--nsecs 60";
     wu->input_files.push_back(
         *make_file(proj, "infile", "in", INPUT_FILE, false)
     );
 #endif
+#ifdef APP_DOCKER_WRAPPER_COPY
+    wu->input_files.push_back(
+        *make_file(proj, "infile", "in", INPUT_FILE, true)
+    );
+#endif
 
     RESULT *result = make_result(av, wu);
-#if 1
+
+////////////// OUTPUT FILES /////////////////
+
+#ifdef APP_WSL_WRAPPER
     result->output_files.push_back(
         *make_file(proj, "outfile", "out", OUTPUT_FILE, false)
     );
 #endif
+#ifdef APP_DOCKER_WRAPPER_COPY
+    result->output_files.push_back(
+        *make_file(proj, "outfile", "out", OUTPUT_FILE, true)
+    );
+#endif
 
     // tell the client not to get work or run benchmarks
     //

client/client_state.cpp

@@ -262,14 +262,16 @@ void CLIENT_STATE::show_host_info() {
                 "-      OS: %s (%s)",
                 wsl.os_name.c_str(), wsl.os_version.c_str()
             );
-            if (wsl.is_docker_available) {
-                msg_printf(NULL, MSG_INFO, "-      Docker version %s",
-                    wsl.docker_version.c_str()
+            if (!wsl.docker_version.empty()) {
+                msg_printf(NULL, MSG_INFO, "-      Docker version %s (%s)",
+                    wsl.docker_version.c_str(),
+                    docker_type_str(wsl.docker_type)
                 );
             }
-            if (wsl.is_docker_compose_available) {
-                msg_printf(NULL, MSG_INFO, "-      Docker compose version %s",
-                    wsl.docker_compose_version.c_str()
+            if (!wsl.docker_compose_version.empty()) {
+                msg_printf(NULL, MSG_INFO, "-      Docker compose version %s (%s)",
+                    wsl.docker_compose_version.c_str(),
+                    docker_type_str(wsl.docker_compose_type)
                 );
             }
         }
@@ -292,14 +294,16 @@ void CLIENT_STATE::show_host_info() {
     }
 
 #ifndef _WIN64
-    if (host_info.docker_available && strlen(host_info.docker_version)) {
-        msg_printf(NULL, MSG_INFO, "Docker version %s found",
-            host_info.docker_version
+    if (strlen(host_info.docker_version)) {
+        msg_printf(NULL, MSG_INFO, "Docker: version %s (%s)",
+            host_info.docker_version,
+            docker_type_str(host_info.docker_type)
         );
     }
-    if (host_info.docker_compose_available && strlen(host_info.docker_compose_version)) {
-        msg_printf(NULL, MSG_INFO, "Docker compose version %s found",
-            host_info.docker_compose_version
+    if (strlen(host_info.docker_compose_version)) {
+        msg_printf(NULL, MSG_INFO, "Docker compose: version %s (%s)",
+            host_info.docker_compose_version,
+            docker_type_str(host_info.docker_compose_type)
         );
     }
 #endif

client/cs_scheduler.cpp

@@ -141,6 +141,9 @@ int CLIENT_STATE::make_scheduler_request(PROJECT* p) {
         g_use_sandbox?1:0,
         p->dont_request_more_work?1:0
     );
+    if (cc_config.dont_use_docker) {
+        fprintf(f, "    <dont_use_docker/>\n");
+    }
     work_fetch.write_request(f, p);
 
     // write client capabilities

client/hostinfo_unix.cpp

@@ -1233,44 +1233,68 @@ int HOST_INFO::get_virtualbox_version() {
     return 0;
 }
 
-// check if docker compose is installed on volunteer's host
-// populates docker compose version and docker_compose_present on success
-bool HOST_INFO::get_docker_compose_info(){
-    FILE* f = popen(command_get_docker_compose_version, "r");
+// check if docker is installed on this host
+// populate docker_version on success
+//
+bool HOST_INFO::get_docker_version_aux(DOCKER_TYPE type){
+    bool ret = false;
+    string cmd = string(docker_cli_prog(type)) + " --version";
+    FILE* f = popen(cmd.c_str(), "r");
     if (f) {
         char buf[256];
         fgets(buf, 256, f);
         std::string version;
-        if (get_docker_compose_version_string(buf, version)) {
-            docker_compose_available = true;
-            safe_strcpy(docker_compose_version, version.c_str());
+        if (get_docker_version_string(type, buf, version)) {
+            safe_strcpy(docker_version, version.c_str());
+            docker_type = type;
+            ret = true;
         }
         pclose(f);
+    }
+    return ret;
+}
+
+bool HOST_INFO::get_docker_version(){
+    if (get_docker_version_aux(PODMAN)) {
+        return true;
+    }
+    if (get_docker_version_aux(DOCKER)) {
         return true;
     }
     return false;
 }
 
-
-// check if docker is installed on volunteer's host
-// populates docker version and docker_present on success
-bool HOST_INFO::get_docker_info(){
-    FILE* f = popen(command_get_docker_version, "r");
+// check if docker compose is installed on this host
+// populate docker_compose_version on success
+//
+bool HOST_INFO::get_docker_compose_version_aux(DOCKER_TYPE type){
+    bool ret = false;
+    string cmd = string(docker_cli_prog(type)) + " compose version";
+    FILE* f = popen(cmd.c_str(), "r");
     if (f) {
         char buf[256];
         fgets(buf, 256, f);
         std::string version;
-        if (get_docker_version_string(buf, version)) {
-            docker_available = true;
-            safe_strcpy(docker_version, version.c_str());
+        if (get_docker_compose_version_string(type, buf, version)) {
+            safe_strcpy(docker_compose_version, version.c_str());
+            docker_compose_type = type;
+            ret = true;
         }
         pclose(f);
+    }
+    return ret;
+}
+
+bool HOST_INFO::get_docker_compose_version(){
+    if (get_docker_compose_version_aux(PODMAN)) {
+        return true;
+    }
+    if (get_docker_compose_version_aux(DOCKER)) {
         return true;
     }
     return false;
 }
 
-
 // get p_vendor, p_model, p_features
 //
 int HOST_INFO::get_cpu_info() {
@@ -1718,10 +1742,8 @@ int HOST_INFO::get_host_info(bool init) {
         get_virtualbox_version();
     }
 
-    if(!cc_config.dont_use_docker){
-        get_docker_info();
-        get_docker_compose_info();
-    }
+    get_docker_version();
+    get_docker_compose_version();
 
     get_cpu_info();
     get_cpu_count();