Skip to content

Commit 2e6e634

Browse files
droctothorpeandreafehrmanowmaschquinnovator
authored
fix(frontend): retrieve archived logs from correct location (#11010)
* fix(frontend): retrieve archived logs from correct location Signed-off-by: droctothorpe <mythicalsunlight@gmail.com> Co-authored-by: andreafehrman <andrea.k.fehrman@vanderbilt.edu> Co-authored-by: owmasch <owenmaschal0598@gmail.com> * Add namespace tag handling and validation Signed-off-by: droctothorpe <mythicalsunlight@gmail.com> Co-authored-by: andreafehrman <andrea.k.fehrman@vanderbilt.edu> Co-authored-by: owmasch <owenmaschal0598@gmail.com> * Remove whitespace from keyFormat Signed-off-by: droctothorpe <mythicalsunlight@gmail.com> Co-authored-by: andreafehrman <andrea.k.fehrman@vanderbilt.edu> Co-authored-by: owmasch <owenmaschal0598@gmail.com> * Update frontend unit tests Signed-off-by: droctothorpe <mythicalsunlight@gmail.com> * Remove superfluous log statements Signed-off-by: droctothorpe <mythicalsunlight@gmail.com> Co-authored-by: quinnovator <jack@jq.codes> * Add link to keyFormat in manifests Signed-off-by: droctothorpe <mythicalsunlight@gmail.com> * Fix workflow parsing for log artifact Signed-off-by: droctothorpe <mythicalsunlight@gmail.com> Co-authored-by: quinnovator <jack@jq.codes> * Fix unit test Signed-off-by: droctothorpe <mythicalsunlight@gmail.com> --------- Signed-off-by: droctothorpe <mythicalsunlight@gmail.com> Co-authored-by: andreafehrman <andrea.k.fehrman@vanderbilt.edu> Co-authored-by: owmasch <owenmaschal0598@gmail.com> Co-authored-by: quinnovator <jack@jq.codes>
1 parent 000ef60 commit 2e6e634

File tree

9 files changed

+204
-101
lines changed

9 files changed

+204
-101
lines changed

‎frontend/server/configs.ts

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,11 @@ export function loadConfigs(argv: string[], env: ProcessEnv): UIConfigs {
9191
ARGO_ARCHIVE_ARTIFACTORY = 'minio',
9292
/** Bucket to retrive logs from */
9393
ARGO_ARCHIVE_BUCKETNAME = 'mlpipeline',
94-
/** Prefix to logs. */
95-
ARGO_ARCHIVE_PREFIX = 'logs',
94+
/** This should match the keyFormat specified in the Argo workflow-controller-configmap.
95+
* It's set here in the manifests:
96+
* https://github.com/kubeflow/pipelines/blob/7b7918ebf8c30e6ceec99283ef20dbc02fdf6a42/manifests/kustomize/third-party/argo/base/workflow-controller-configmap-patch.yaml#L28
97+
*/
98+
ARGO_KEYFORMAT = 'artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}',
9699
/** Should use server API for log streaming? */
97100
STREAM_LOGS_FROM_SERVER_API = 'false',
98101
/** The main container name of a pod where logs are retrieved */
@@ -127,7 +130,7 @@ export function loadConfigs(argv: string[], env: ProcessEnv): UIConfigs {
127130
archiveArtifactory: ARGO_ARCHIVE_ARTIFACTORY,
128131
archiveBucketName: ARGO_ARCHIVE_BUCKETNAME,
129132
archiveLogs: asBool(ARGO_ARCHIVE_LOGS),
130-
archivePrefix: ARGO_ARCHIVE_PREFIX,
133+
keyFormat: ARGO_KEYFORMAT,
131134
},
132135
pod: {
133136
logContainerName: POD_LOG_CONTAINER_NAME,
@@ -253,7 +256,7 @@ export interface ArgoConfigs {
253256
archiveLogs: boolean;
254257
archiveArtifactory: string;
255258
archiveBucketName: string;
256-
archivePrefix: string;
259+
keyFormat: string;
257260
}
258261
export interface ServerConfigs {
259262
basePath: string;

‎frontend/server/handlers/pod-logs.ts

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,21 +39,21 @@ export function getPodLogsHandler(
3939
},
4040
podLogContainerName: string,
4141
): Handler {
42-
const { archiveLogs, archiveArtifactory, archiveBucketName, archivePrefix = '' } = argoOptions;
42+
const { archiveLogs, archiveArtifactory, archiveBucketName, keyFormat } = argoOptions;
4343

44-
// get pod log from the provided bucket and prefix.
44+
// get pod log from the provided bucket and keyFormat.
4545
const getPodLogsStreamFromArchive = toGetPodLogsStream(
4646
createPodLogsMinioRequestConfig(
4747
archiveArtifactory === 'minio' ? artifactsOptions.minio : artifactsOptions.aws,
4848
archiveBucketName,
49-
archivePrefix,
49+
keyFormat,
5050
),
5151
);
5252

5353
// get the pod log stream (with fallbacks).
5454
const getPodLogsStream = composePodLogsStreamHandler(
55-
(podName: string, namespace?: string) => {
56-
return getPodLogsStreamFromK8s(podName, namespace, podLogContainerName);
55+
(podName: string, createdAt: string, namespace?: string) => {
56+
return getPodLogsStreamFromK8s(podName, createdAt, namespace, podLogContainerName);
5757
},
5858
// if archive logs flag is set, then final attempt will try to retrieve the artifacts
5959
// from the bucket and prefix provided in the config. Otherwise, only attempts
@@ -69,13 +69,14 @@ export function getPodLogsHandler(
6969
return;
7070
}
7171
const podName = decodeURIComponent(req.query.podname);
72+
const createdAt = decodeURIComponent(req.query.createdat);
7273

7374
// This is optional.
7475
// Note decodeURIComponent(undefined) === 'undefined', so I cannot pass the argument directly.
7576
const podNamespace = decodeURIComponent(req.query.podnamespace || '') || undefined;
7677

7778
try {
78-
const stream = await getPodLogsStream(podName, podNamespace);
79+
const stream = await getPodLogsStream(podName, createdAt, podNamespace);
7980
stream.on('error', err => {
8081
if (
8182
err?.message &&

‎frontend/server/workflow-helper.test.ts

Lines changed: 56 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -39,40 +39,49 @@ describe('workflow-helper', () => {
3939
describe('composePodLogsStreamHandler', () => {
4040
it('returns the stream from the default handler if there is no errors.', async () => {
4141
const defaultStream = new PassThrough();
42-
const defaultHandler = jest.fn((_podName: string, _namespace?: string) =>
42+
const defaultHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) =>
4343
Promise.resolve(defaultStream),
4444
);
45-
const stream = await composePodLogsStreamHandler(defaultHandler)('podName', 'namespace');
46-
expect(defaultHandler).toBeCalledWith('podName', 'namespace');
45+
const stream = await composePodLogsStreamHandler(defaultHandler)(
46+
'podName',
47+
'2024-08-13',
48+
'namespace',
49+
);
50+
expect(defaultHandler).toBeCalledWith('podName', '2024-08-13', 'namespace');
4751
expect(stream).toBe(defaultStream);
4852
});
4953

5054
it('returns the stream from the fallback handler if there is any error.', async () => {
5155
const fallbackStream = new PassThrough();
52-
const defaultHandler = jest.fn((_podName: string, _namespace?: string) =>
56+
const defaultHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) =>
5357
Promise.reject('unknown error'),
5458
);
55-
const fallbackHandler = jest.fn((_podName: string, _namespace?: string) =>
59+
const fallbackHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) =>
5660
Promise.resolve(fallbackStream),
5761
);
5862
const stream = await composePodLogsStreamHandler(defaultHandler, fallbackHandler)(
5963
'podName',
64+
'2024-08-13',
6065
'namespace',
6166
);
62-
expect(defaultHandler).toBeCalledWith('podName', 'namespace');
63-
expect(fallbackHandler).toBeCalledWith('podName', 'namespace');
67+
expect(defaultHandler).toBeCalledWith('podName', '2024-08-13', 'namespace');
68+
expect(fallbackHandler).toBeCalledWith('podName', '2024-08-13', 'namespace');
6469
expect(stream).toBe(fallbackStream);
6570
});
6671

6772
it('throws error if both handler and fallback fails.', async () => {
68-
const defaultHandler = jest.fn((_podName: string, _namespace?: string) =>
73+
const defaultHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) =>
6974
Promise.reject('unknown error for default'),
7075
);
71-
const fallbackHandler = jest.fn((_podName: string, _namespace?: string) =>
76+
const fallbackHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) =>
7277
Promise.reject('unknown error for fallback'),
7378
);
7479
await expect(
75-
composePodLogsStreamHandler(defaultHandler, fallbackHandler)('podName', 'namespace'),
80+
composePodLogsStreamHandler(defaultHandler, fallbackHandler)(
81+
'podName',
82+
'2024-08-13',
83+
'namespace',
84+
),
7685
).rejects.toEqual('unknown error for fallback');
7786
});
7887
});
@@ -82,7 +91,7 @@ describe('workflow-helper', () => {
8291
const mockedGetPodLogs: jest.Mock = getPodLogs as any;
8392
mockedGetPodLogs.mockResolvedValueOnce('pod logs');
8493

85-
const stream = await getPodLogsStreamFromK8s('podName', 'namespace');
94+
const stream = await getPodLogsStreamFromK8s('podName', '', 'namespace');
8695
expect(mockedGetPodLogs).toBeCalledWith('podName', 'namespace', 'main');
8796
expect(stream.read().toString()).toBe('pod logs');
8897
});
@@ -101,24 +110,34 @@ describe('workflow-helper', () => {
101110
client,
102111
key: 'folder/key',
103112
};
104-
const createRequest = jest.fn((_podName: string, _namespace?: string) =>
113+
const createRequest = jest.fn((_podName: string, _createdAt: string, _namespace?: string) =>
105114
Promise.resolve(configs),
106115
);
107-
const stream = await toGetPodLogsStream(createRequest)('podName', 'namespace');
116+
const stream = await toGetPodLogsStream(createRequest)('podName', '2024-08-13', 'namespace');
108117
expect(mockedClientGetObject).toBeCalledWith('bucket', 'folder/key');
109118
});
110119
});
111120

112121
describe('createPodLogsMinioRequestConfig', () => {
113122
it('returns a MinioRequestConfig factory with the provided minioClientOptions, bucket, and prefix.', async () => {
114123
const mockedClient: jest.Mock = MinioClient as any;
115-
const requestFunc = await createPodLogsMinioRequestConfig(minioConfig, 'bucket', 'prefix');
116-
const request = await requestFunc('workflow-name-abc', 'namespace');
124+
const requestFunc = await createPodLogsMinioRequestConfig(
125+
minioConfig,
126+
'bucket',
127+
'artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}',
128+
);
129+
const request = await requestFunc(
130+
'workflow-name-system-container-impl-foo',
131+
'2024-08-13',
132+
'namespace',
133+
);
117134

118135
expect(mockedClient).toBeCalledWith(minioConfig);
119136
expect(request.client).toBeInstanceOf(MinioClient);
120137
expect(request.bucket).toBe('bucket');
121-
expect(request.key).toBe('prefix/workflow-name/workflow-name-abc/main.log');
138+
expect(request.key).toBe(
139+
'artifacts/workflow-name/2024/08/13/workflow-name-system-container-impl-foo/main.log',
140+
);
122141
});
123142
});
124143

@@ -128,31 +147,28 @@ describe('workflow-helper', () => {
128147
apiVersion: 'argoproj.io/v1alpha1',
129148
kind: 'Workflow',
130149
status: {
150+
artifactRepositoryRef: {
151+
artifactRepository: {
152+
archiveLogs: true,
153+
s3: {
154+
accessKeySecret: { key: 'accessKey', name: 'accessKeyName' },
155+
bucket: 'bucket',
156+
endpoint: 'minio-service.kubeflow',
157+
insecure: true,
158+
key:
159+
'prefix/workflow-name/workflow-name-system-container-impl-abc/some-artifact.csv',
160+
secretKeySecret: { key: 'secretKey', name: 'secretKeyName' },
161+
},
162+
},
163+
},
131164
nodes: {
132165
'workflow-name-abc': {
133166
outputs: {
134167
artifacts: [
135168
{
136-
name: 'some-artifact.csv',
137-
s3: {
138-
accessKeySecret: { key: 'accessKey', name: 'accessKeyName' },
139-
bucket: 'bucket',
140-
endpoint: 'minio-service.kubeflow',
141-
insecure: true,
142-
key: 'prefix/workflow-name/workflow-name-abc/some-artifact.csv',
143-
secretKeySecret: { key: 'secretKey', name: 'secretKeyName' },
144-
},
145-
},
146-
{
147-
archiveLogs: true,
148-
name: 'main.log',
169+
name: 'main-logs',
149170
s3: {
150-
accessKeySecret: { key: 'accessKey', name: 'accessKeyName' },
151-
bucket: 'bucket',
152-
endpoint: 'minio-service.kubeflow',
153-
insecure: true,
154-
key: 'prefix/workflow-name/workflow-name-abc/main.log',
155-
secretKeySecret: { key: 'secretKey', name: 'secretKeyName' },
171+
key: 'prefix/workflow-name/workflow-name-system-container-impl-abc/main.log',
156172
},
157173
},
158174
],
@@ -174,7 +190,10 @@ describe('workflow-helper', () => {
174190
mockedClientGetObject.mockResolvedValueOnce(objStream);
175191
objStream.end('some fake logs.');
176192

177-
const stream = await getPodLogsStreamFromWorkflow('workflow-name-abc');
193+
const stream = await getPodLogsStreamFromWorkflow(
194+
'workflow-name-system-container-impl-abc',
195+
'2024-07-09',
196+
);
178197

179198
expect(mockedGetArgoWorkflow).toBeCalledWith('workflow-name');
180199

@@ -193,7 +212,7 @@ describe('workflow-helper', () => {
193212
expect(mockedClientGetObject).toBeCalledTimes(1);
194213
expect(mockedClientGetObject).toBeCalledWith(
195214
'bucket',
196-
'prefix/workflow-name/workflow-name-abc/main.log',
215+
'prefix/workflow-name/workflow-name-system-container-impl-abc/main.log',
197216
);
198217
});
199218
});

0 commit comments

Comments
 (0)