Skip to content

Commit 5b9ce6b

Browse files
jakesmithjakesmith
committed
Merge remote-tracking branch 'origin/candidate-9.10.x'
Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com> # Conflicts: # helm/hpcc/Chart.yaml # helm/hpcc/templates/_helpers.tpl # version.cmake
2 parents 009eaa5 + f18e096 commit 5b9ce6b

File tree

25 files changed

+434
-301
lines changed

25 files changed

+434
-301
lines changed

‎.github/actions/hpcc4j-esp-regression-suite/action.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ runs:
3030
cd ./HPCC4j
3131
mvn -B --activate-profiles jenkins-on-demand -Dmaven.gpg.skip=true \
3232
-Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=true -Dhpccconn=https://eclwatch.default:8010 \
33-
-Dwssqlconn=https://sql2ecl.default:8510 -DHPCC30117=open -pl '!dfsclient' install
33+
-Dwssqlconn=https://sql2ecl.default:8510 -DHPCC30117=open -pl commons-hpcc -pl wsclient install
3434
3535
- name: Process errors
3636
env:

‎.github/workflows/build-docker.yml

+25-3
Original file line numberDiff line numberDiff line change
@@ -108,22 +108,27 @@ jobs:
108108
run: |
109109
branch_label_1=${{ github.base_ref }}
110110
branch_label_2=$(echo ${{ github.ref }} | cut -d'/' -f3)
111-
echo "branch_label=${branch_label_1:-$branch_label_2}" >> $GITHUB_OUTPUT
111+
branch_label=${branch_label_1:-$branch_label_2}
112+
echo "branch_label=$branch_label" >> $GITHUB_OUTPUT
112113
vcpkg_sha_short=$(git rev-parse --short=8 HEAD)
113114
echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT
114-
docker_build_label=hpccsystems/platform-build-base-${{ inputs.os }}
115+
docker_build_label=hpccsystems/platform-build-${{ inputs.os }}
115116
echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT
117+
echo "docker_tag_candidate_base=$docker_build_label:$branch_label" >> $GITHUB_OUTPUT
116118
117119
- name: Print vars
118120
shell: "bash"
119121
run: |
120122
echo "${{ toJSON(steps.vars.outputs) }}"
121123
122-
- uses: docker/setup-buildx-action@v2
124+
- name: Set up Docker Buildx
125+
id: buildx
126+
uses: docker/setup-buildx-action@v3
123127

124128
- name: Pull Build Image
125129
run: |
126130
docker pull ${{ steps.vars.outputs.docker_tag }} || true
131+
docker pull ${{ steps.vars.outputs.docker_tag_candidate_base }} || true
127132
128133
- uses: hendrikmuhs/ccache-action@v1.2
129134
with:
@@ -135,6 +140,23 @@ jobs:
135140
docker-${{ inputs.os }}-${{ inputs.build-type }}-
136141
docker-${{ inputs.os }}-
137142
143+
- name: Docker build image
144+
uses: docker/build-push-action@v5
145+
with:
146+
builder: ${{ steps.buildx.outputs.name }}
147+
file: HPCC-Platform/dockerfiles/vcpkg/${{ inputs.os }}.dockerfile
148+
context: HPCC-Platform/dockerfiles/vcpkg/.
149+
push: false
150+
load: true
151+
build-args: |
152+
VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }}
153+
tags: |
154+
${{ steps.vars.outputs.docker_tag }}
155+
${{ steps.vars.outputs.docker_tag_candidate_base }}
156+
cache-from: |
157+
type=registry,ref=${{ steps.vars.outputs.docker_tag }}
158+
type=registry,ref=${{ steps.vars.outputs.docker_tag_candidate_base }}
159+
138160
- name: CMake Configure and Build
139161
shell: "bash"
140162
run: |

‎dali/sasha/saxref.cpp

+1-2
Original file line numberDiff line numberDiff line change
@@ -2363,7 +2363,6 @@ class CSashaExpiryServer: public ISashaServer, public Thread
23632363
if (stopped)
23642364
break;
23652365
const char *lfn = expirylist.item(i);
2366-
PROGLOG(LOGPFX2 "Deleting %s",lfn);
23672366
try
23682367
{
23692368
/* NB: 0 timeout, meaning fail and skip, if there is any locking contention.
@@ -2374,7 +2373,7 @@ class CSashaExpiryServer: public ISashaServer, public Thread
23742373
}
23752374
catch (IException *e) // may want to just detach if fails
23762375
{
2377-
EXCLOG(e, LOGPFX2 "remove");
2376+
OWARNLOG(e, LOGPFX2 "remove");
23782377
e->Release();
23792378
}
23802379
}

‎docs/EN_US/ECLLanguageReference/ECLR_mods/Templ-OPTION.xml

+11
Original file line numberDiff line numberDiff line change
@@ -1193,6 +1193,17 @@
11931193
statement to indicate how many TRACE statement to write to log
11941194
file. See <link linkend="TRACE">TRACE</link>.</entry>
11951195
</row>
1196+
1197+
<row>
1198+
<entry><emphasis>maxlogdetail</emphasis></entry>
1199+
1200+
<entry/>
1201+
1202+
<entry>Overrides the the default logging level for a single
1203+
workunit. This allows logging levels to be set to a low level by
1204+
default, but allow jobs to be resubmitted with a higher logging
1205+
level for investigation.</entry>
1206+
</row>
11961207
</tbody>
11971208
</tgroup>
11981209
</informaltable></para>

‎docs/EN_US/HPCCDataHandling/DH-Mods/DH-Mod1.xml

+96-29
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<?xml version="1.0" encoding="UTF-8"?>
2-
<!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
2+
<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
33
"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
44
<chapter>
55
<title>Data Handling Components</title>
@@ -34,6 +34,73 @@
3434
that server to enable file sprays and desprays.</para>
3535
</sect1>
3636

37+
<sect1 id="FileMovement">
38+
<title>File Movement</title>
39+
40+
<para>The ESP file service can request whether an import (spray), despray,
41+
or copy command is a <emphasis role="bold">push</emphasis> or a <emphasis
42+
role="bold">pull</emphasis> command.</para>
43+
44+
<para>These processes run as DFU workunits and communicate with service
45+
components, either running as dynamically launched ftslave processes (the
46+
default behavior on Bare-Metal deployments), or as tasks running on a
47+
dafilesrv service (the only option on a Containerized deployment).</para>
48+
49+
<itemizedlist>
50+
<listitem>
51+
<para><emphasis role="bold">pull</emphasis> means that the commands
52+
will notionally run per target part and pull from the source.</para>
53+
</listitem>
54+
55+
<listitem>
56+
<para><emphasis role="bold">push</emphasis> means the commands will
57+
notionally run per source part and push to the target.</para>
58+
</listitem>
59+
</itemizedlist>
60+
61+
<para>In Bare-Metal deployments, where the compute is coupled to the data,
62+
ftslave processes are launched on either the source or the target nodes
63+
depending on whether pushing or pulling.</para>
64+
65+
<para>In a Containerized system, the commands always run on a dedicated
66+
dafilesrv spray-service where neither the source nor the target are
67+
local.</para>
68+
69+
<para>The default setting in a bare-metal system is useFtSlave=true, an
70+
option of dfuserver, but can be configured off so Bare-Metal will use
71+
dafilesrv as a service, too.</para>
72+
73+
<para>It is generally recommended to leave the decision to whether to pull
74+
or push up to the platform. In Bare-Metal, it may decide that it has to
75+
push because running ftslave on the target side is not supported such as
76+
when the the Landing Zone is a Windows server. It may also choose to pull
77+
when it concludes it is more efficient to do so. In a Containerized
78+
system, some target planes may not support multiple processes writing
79+
concurrently, and therefore pull is forced. If you implicitly request pull
80+
or push and the platform cannot satisfy the request, it will warn and
81+
continue with the method that is supported.</para>
82+
83+
<para>In addition, the <emphasis role="bold">noCommon</emphasis> option is
84+
available. The noCommon option prevents the job/dfuserver from "commoning
85+
up" the individual partition commands into a single request. Commoning up
86+
means a single ftslave or service command will process multiple partitions
87+
sequentially. Not commoning up, means multiple concurrent partitions, that
88+
would otherwise be grouped, will operate in parallel as individual
89+
ftslaves or individual spray service commands.</para>
90+
91+
<para>In Bare-Metal, noCommon is on by default, and will mean that any
92+
partition command that is destined for the same host, that would otherwise
93+
be grouped into a single command, will be left as multiple concurrent
94+
commands. For example, if spraying a file to a Thor cluster in Bare-Metal,
95+
and the Thor group has several partitions per node (because
96+
slavesPerNode&gt;1), then with the default noCommon=true and pull mode,
97+
each of the multiple partition pull commands for each node would run in
98+
parallel.</para>
99+
100+
<para>In Containerized deployments, noCommon is currently not supported
101+
and will be ignored.</para>
102+
</sect1>
103+
37104
<sect1 id="Working_with_a_data_file">
38105
<title>Working with data files</title>
39106

@@ -66,14 +133,14 @@
66133
<?dbfo keep-together="always"?>
67134

68135
<tgroup cols="2">
69-
<colspec colwidth="49.50pt" />
136+
<colspec colwidth="49.50pt"/>
70137

71-
<colspec />
138+
<colspec/>
72139

73140
<tbody>
74141
<row>
75142
<entry><inlinegraphic
76-
fileref="../../images/caution.png" /></entry>
143+
fileref="../../images/caution.png"/></entry>
77144

78145
<entry>Your IP address could be different from the ones
79146
provided in the example images. Please use the IP address
@@ -99,7 +166,7 @@
99166
<mediaobject>
100167
<imageobject>
101168
<imagedata fileref="../../images/LZimg03-1.jpg"
102-
vendor="eclwatchSS" />
169+
vendor="eclwatchSS"/>
103170
</imageobject>
104171
</mediaobject>
105172
</figure>A system file dialog opens. Browse and select the file to
@@ -112,7 +179,7 @@
112179
<mediaobject>
113180
<imageobject>
114181
<imagedata fileref="../../images/LZimg04.jpg"
115-
vendor="eclwatchSS" />
182+
vendor="eclwatchSS"/>
116183
</imageobject>
117184
</mediaobject>
118185
</figure></para>
@@ -146,9 +213,9 @@
146213

147214
<para><informaltable colsep="1" rowsep="1">
148215
<tgroup cols="2">
149-
<colspec colwidth="80pt" />
216+
<colspec colwidth="80pt"/>
150217

151-
<colspec colwidth="100pt" />
218+
<colspec colwidth="100pt"/>
152219

153220
<tbody>
154221
<row>
@@ -235,7 +302,7 @@
235302

236303
<mediaobject>
237304
<imageobject>
238-
<imagedata fileref="../../images/IMDB_01.jpg" />
305+
<imagedata fileref="../../images/IMDB_01.jpg"/>
239306
</imageobject>
240307
</mediaobject>
241308
</figure></para>
@@ -291,7 +358,7 @@
291358
entries in the Dali Server's Distributed File System.</para>
292359

293360
<para><graphic fileref="../../images/DHMan-3.jpg"
294-
vendor="eclwatchSS" /></para>
361+
vendor="eclwatchSS"/></para>
295362

296363
<para>From this page, you can despray or copy any file.</para>
297364
</listitem>
@@ -328,9 +395,9 @@
328395

329396
<para><informaltable colsep="0" frame="none" rowsep="0">
330397
<tgroup cols="2">
331-
<colspec colwidth="122.40pt" />
398+
<colspec colwidth="122.40pt"/>
332399

333-
<colspec colwidth="333.00pt" />
400+
<colspec colwidth="333.00pt"/>
334401

335402
<tbody>
336403
<row>
@@ -474,9 +541,9 @@
474541

475542
<para><informaltable colsep="0" frame="none" rowsep="0">
476543
<tgroup cols="2">
477-
<colspec colwidth="1*" />
544+
<colspec colwidth="1*"/>
478545

479-
<colspec colwidth="3*" />
546+
<colspec colwidth="3*"/>
480547

481548
<tbody>
482549
<row>
@@ -680,9 +747,9 @@
680747

681748
<para><informaltable colsep="0" frame="none" rowsep="0">
682749
<tgroup cols="2">
683-
<colspec colwidth="122.40pt" />
750+
<colspec colwidth="122.40pt"/>
684751

685-
<colspec colwidth="333.00pt" />
752+
<colspec colwidth="333.00pt"/>
686753

687754
<tbody>
688755
<row>
@@ -838,9 +905,9 @@
838905

839906
<para><informaltable colsep="0" frame="none" rowsep="0">
840907
<tgroup cols="2">
841-
<colspec colwidth="122.40pt" />
908+
<colspec colwidth="122.40pt"/>
842909

843-
<colspec colwidth="333.00pt" />
910+
<colspec colwidth="333.00pt"/>
844911

845912
<tbody>
846913
<row>
@@ -998,9 +1065,9 @@
9981065

9991066
<para><informaltable colsep="0" frame="none" rowsep="0">
10001067
<tgroup cols="2">
1001-
<colspec colwidth="122.40pt" />
1068+
<colspec colwidth="122.40pt"/>
10021069

1003-
<colspec colwidth="333.00pt" />
1070+
<colspec colwidth="333.00pt"/>
10041071

10051072
<tbody>
10061073
<row>
@@ -1145,9 +1212,9 @@
11451212

11461213
<para><informaltable colsep="0" frame="none" rowsep="0">
11471214
<tgroup cols="2">
1148-
<colspec colwidth="122.40pt" />
1215+
<colspec colwidth="122.40pt"/>
11491216

1150-
<colspec colwidth="333.00pt" />
1217+
<colspec colwidth="333.00pt"/>
11511218

11521219
<tbody>
11531220
<row>
@@ -1271,7 +1338,7 @@
12711338
button.</para>
12721339

12731340
<para><graphic fileref="../../images/DHMan-3a.jpg"
1274-
vendor="eclwatchSS" /></para>
1341+
vendor="eclwatchSS"/></para>
12751342
</listitem>
12761343

12771344
<listitem>
@@ -1280,9 +1347,9 @@
12801347

12811348
<para><informaltable colsep="0" frame="none" rowsep="0">
12821349
<tgroup cols="2">
1283-
<colspec align="left" colwidth="122.40pt" />
1350+
<colspec align="left" colwidth="122.40pt"/>
12841351

1285-
<colspec colwidth="333.00pt" />
1352+
<colspec colwidth="333.00pt"/>
12861353

12871354
<tbody>
12881355
<row>
@@ -1373,9 +1440,9 @@
13731440

13741441
<informaltable colsep="0" frame="none" rowsep="0">
13751442
<tgroup cols="2">
1376-
<colspec colwidth="122.40pt" />
1443+
<colspec colwidth="122.40pt"/>
13771444

1378-
<colspec colwidth="333.00pt" />
1445+
<colspec colwidth="333.00pt"/>
13791446

13801447
<tbody>
13811448
<row>
@@ -1499,9 +1566,9 @@
14991566

15001567
<informaltable colsep="0" frame="none" rowsep="0">
15011568
<tgroup cols="2">
1502-
<colspec colwidth="122.40pt" />
1569+
<colspec colwidth="122.40pt"/>
15031570

1504-
<colspec colwidth="333.00pt" />
1571+
<colspec colwidth="333.00pt"/>
15051572

15061573
<tbody>
15071574
<row>

0 commit comments

Comments
 (0)