PS-652_indexer-wrong-dispatch-level2 (#440)

* fix empty values dispatch & multi->mono order

* twig context : add attributes to record, add helpers to gets status-bits and subdefs.
add doc.

databox/indexer/doc/conf_phraseanet.md

@@ -107,58 +107,73 @@ Each path is a **twig** expressions that must generate databox path(s), dependin
 If the asset is to be copied in many places (paths), the twig must generate **one line per path**.
 
 - ### `copyTo` Twig context :
-
-    - `record` record object
-
-    - `record.title` : _todo_
-
-    - `record.uuid` : _todo_
-
-    - `record.original_name` : _todo_
-
-    - `record.status` : _todo_
-
-    - `record.getMetadata(<fieldName> [,<default>])` : metadata object, with default value(s) if the field is not set for this record.
-
-    - `record.getMetadata(...).value` : The mono-value (if the field is multi-value : concat values with " ; ").
-
-    - `record.getMetadata(...).values` : The multi-values as array (if the field is mono-value : array with a single value).
-
-        - e.g. 1: Two levels dispatch with unique destination (mono-value fields):
 
-            ```json lines
-            ...
-            "copyTo": [
-              "/classification/{{record.getMetadata('Category', 'unknown_category').value | escapePath}}/{{record.getMetadata('SubCategory', 'unknown_subcategory').value | escapePath}}"
-            ]
-            ...
-            ```
-
-        - e.g. 2: Multiple destinations (multi-values field):
-
-            ```json lines
-            ...
-            "copyTo": [
-              "{% for s in record.getMetadata('Keywords', 'no_keyword').values %}/classification/{{ s | escapePath }}\n{% endfor %}"
-            ]
-            ...
-            ```
-          note: The `\n` is used to output one line (= one path) per keyword.
-
-          note: The default value "no_keyword" is a must-have, because if the record had no keyword, it would not be copied anywhere.
-
-        - e.g. 3: multiple destinations :
-
-          To dispatch the records in many "classification" places, one can set many `copyTo` setting.
-            ```json lines
-            ...
-            "copyTo": [
-              "/classification/author/{{record.getMetadata('Author', 'unknown_author').value | escapePath}},
-              "/classification/category/{{record.getMetadata('Category', 'unknown_category').value | escapePath}},
-              "/classification/year/{{record.getMetadata('Date', '').value is empty ? 'unknown_date' : {{record.getMetadata('Date').value | date('Y')}}
-            ]
-            ...
-            ```
+    - `record`: record object
+      - `record.record_id` : string
+      - `record.resource_id` : string
+      - `record.databox_id` : string
+      - `record.base_id` : string
+      - `record.uuid` : string
+      - `record.title` : string
+      - `record.original_name` : string
+      - `record.mime_type` : string
+      - `record.created_on` : string
+      - `record.updated_on` : string
+      - `record.status` : status[] ***use `getStatus()` method***
+      - `record.getStatus(<bit> [, <valueIfTrue> [, <valueIfFalse>]])` : boolean ; Value of sb <bit> (4...63).
+        Boolean value can be replaced by string value(s) `valueIf...` 
+      - `record.subdef` : subdef[] ***use `getSubdef()` method***
+      - `record.getSubdef(<name>)` : subdef object
+        - `record.getSubdef(...).height` : number
+        - `record.getSubdef(...).width` : number
+        - `record.getSubdef(...).filesize` : number
+        - `record.getSubdef(...).player_type` : string
+        - `record.getSubdef(...).mime_type` : number
+        - `record.getSubdef(...).created_on` : string
+        - `record.getSubdef(...).updated_on` : string
+        - `record.getSubdef(...).url` : string
+        - `record.getSubdef(...).permalink` : permalink object
+          - `record.getSubdef(...).permalink.url` : string
+      - `record.metadata` : metata[] ***use `getMetadata()` method***
+      - `record.getMetadata(<fieldName> [,<default>])` : metadata object, with default value(s) if the field is not set for this record.
+        - `record.getMetadata(...).value` : The mono-value (if the field is multi-value : concat values with " ; ").
+        - `record.getMetadata(...).values` : The multi-values as array (if the field is mono-value : array with a single value).
+
+    - e.g. 1: Two levels dispatch with unique destination (mono-value fields):
+
+        ```json lines
+        ...
+        "copyTo": [
+          "/classification/{{record.getMetadata('Category', 'unknown_category').value | escapePath}}/{{record.getMetadata('SubCategory', 'unknown_subcategory').value | escapePath}}"
+        ]
+        ...
+        ```
+
+    - e.g. 2: Multiple destinations (multi-values field):
+
+        ```json lines
+        ...
+        "copyTo": [
+          "{% for s in record.getMetadata('Keywords', 'no_keyword').values %}/classification/{{ s | escapePath }}\n{% endfor %}"
+        ]
+        ...
+        ```
+      note: The `\n` is used to output one line (= one path) per keyword.
+
+      note: The default value "no_keyword" is a must-have, because if the record had no keyword, it would not be copied anywhere.
+
+    - e.g. 3: multiple destinations :
+
+      To dispatch the records in many "classification" places, one can set multiple `copyTo` settings.
+        ```json lines
+        ...
+        "copyTo": [
+          "/classification/author/{{record.getMetadata('Author', 'unknown_author').value | escapePath}},
+          "/classification/category/{{record.getMetadata('Category', 'unknown_category').value | escapePath}},
+          "/classification/year/{{record.getMetadata('Date', '').value is empty ? 'unknown_date' : {{record.getMetadata('Date').value | date('Y')}}
+        ]
+        ...
+        ```
 
 ## `fieldMap` 
 Map (key=AttributeDefinition name) of attributes to create / import.
@@ -214,3 +229,5 @@ _twig context technical note_:
 
 To prevent twig to crash if a field doest not exists in a record (when trying to access a property like `.value`),
 `getMetadata(...)` will return a "fake" empty metadata object.
+
+Same method applies for subdefs: `record.getSubdef('missingSubdef').permalink.url` will return null. 

databox/indexer/src/handlers/phraseanet/CPhraseanetRecord.ts

@@ -1,11 +1,12 @@
 import PhraseanetClient from "./phraseanetClient";
 import {
     PhraseanetStatusBit,
-    SubDef,
+    PhraseanetSubdef,
     PhraseanetRecord,
     PhraseanetStory,
 } from "./types";
 import {CPhraseanetMetadata} from "./CPhraseanetMetadata";
+import {CPhraseanetSubdef} from "./CPhraseanetSubdef";
 
 class CPhraseanetRecordBase {
     client:PhraseanetClient = {} as PhraseanetClient;
@@ -15,9 +16,13 @@ class CPhraseanetRecordBase {
     uuid: string = "";
     title: string = "";
     original_name: string = "";
-    subdefs: SubDef[] = [];
+    mime_type: string = "";
+    created_on: string = "";
+    updated_on: string = "";
+    subdefs: PhraseanetSubdef[] = [];
     metadata: Record<string, CPhraseanetMetadata> = {};
     status: PhraseanetStatusBit[] = [];
+    private csubdefs: Record<string, CPhraseanetSubdef> = {};
 
     async getMetadata(fieldName: string, defaultValue?: string): Promise<CPhraseanetMetadata> {
         if(this.metadata[fieldName]) {
@@ -30,6 +35,21 @@ class CPhraseanetRecordBase {
         return CPhraseanetMetadata.NullMetadata;
     }
 
+    async getStatus(bit: number, valueTrue?: string, valueFalse?: string): Promise<string|boolean> {
+        const vTrue: string|boolean = valueTrue ?? true;
+        const vFalse: string|boolean = valueFalse ?? false;
+        for(const s of this.status) {
+            if(s.bit === bit) {
+                return s.state ? vTrue : vFalse;
+            }
+        }
+        return vFalse;
+    }
+
+    async getSubdef(name: string): Promise<CPhraseanetSubdef> {
+        return this.csubdefs[name] ?? CPhraseanetSubdef.NullSubdef;
+    }
+
     constructor(r:PhraseanetRecord|PhraseanetStory, client:PhraseanetClient) {
         this.client = client;
         this.resource_id = r.resource_id;
@@ -40,15 +60,31 @@ class CPhraseanetRecordBase {
         this.original_name = r.original_name;
         this.subdefs = r.subdefs;
         this.status = r.status;
+        this.mime_type = r.mime_type;
+        this.created_on = r.created_on;
+        this.updated_on = r.updated_on;
         r.metadata.map((m) => {
-            if(!this.metadata[m.name]) {
-                this.metadata[m.name] = CPhraseanetMetadata.fromTPhraseanetMetadata(m);
+            if(m.value.trim() !== '') {
+                if (!this.metadata[m.name]) {
+                    this.metadata[m.name] = CPhraseanetMetadata.fromTPhraseanetMetadata(m);
+                }
+                this.metadata[m.name].values.push(m.value);
             }
-            this.metadata[m.name].values.push(m.value);
         })
+
         for(const k in this.metadata) {
+            this.metadata[k].values.sort(
+                (a, b) => {
+                    a = a.toLowerCase(); b = b.toLowerCase();
+                    return a < b ? -1 : (a > b ? 1 : 0);
+                }
+            );
             this.metadata[k].value = this.metadata[k].values.join(' ; ')
         }
+
+        r.subdefs.map((s) => {
+            this.csubdefs[s.name] = new CPhraseanetSubdef(s);
+        });
     }
 }
 

databox/indexer/src/handlers/phraseanet/CPhraseanetSubdef.ts

@@ -0,0 +1,35 @@
+import {PhraseanetSubdef} from "./types";
+
+export class CPhraseanetSubdef {
+    name?: string;
+    height?: number;
+    width?: number;
+    filesize?: number;
+    player_type?: string;
+    mime_type?: string;
+    created_on?: string;
+    updated_on?: string;
+    url?: string;
+    permalink?: {
+        url?: string;
+    };
+
+    static NullSubdef = new CPhraseanetSubdef();
+
+    constructor(s?: PhraseanetSubdef) {
+        if(s) {
+            this.name = s.name;
+            this.height = s.height;
+            this.width = s.width;
+            this.filesize = s.filesize;
+            this.player_type = s.player_type;
+            this.mime_type = s.mime_type;
+            this.created_on = s.created_on;
+            this.updated_on = s.updated_on;
+            this.url = s.url;
+            this.permalink = s.permalink;
+        }
+    }
+
+}
+

databox/indexer/src/handlers/phraseanet/indexer.ts

@@ -215,8 +215,8 @@ export const phraseanetIndexer: IndexIterator<PhraseanetConfig> =
                 classIndex[rc.name] = rc.id;
             });
 
-            const subDefs = await client.getSubDefinitions(databox.databox_id);
-            for (const sd of subDefs) {
+            const subdefs = await client.getSubdefsStruct(databox.databox_id);
+            for (const sd of subdefs) {
                 if (!classIndex[sd.class]) {
                     logger.info(`Creating rendition class "${sd.class}" `);
                     classIndex[sd.class] =

databox/indexer/src/handlers/phraseanet/phraseanetClient.ts

@@ -6,7 +6,7 @@ import {
     PhraseanetDatabox,
     PhraseanetMetaStruct,
     PhraseanetStatusBitStruct,
-    PhraseanetSubDef,
+    PhraseanetSubdefStruct,
     PhraseanetRecord,
     PhraseanetStory
 } from './types';
@@ -189,11 +189,11 @@ export default class PhraseanetClient {
         return res.data.response.status;
     }
 
-    async getSubDefinitions(databoxId?: string): Promise<PhraseanetSubDef[]> {
+    async getSubdefsStruct(databoxId?: string): Promise<PhraseanetSubdefStruct[]> {
         const dbid = typeof databoxId !== 'undefined' ? '/' + databoxId : '';
         const res = await this.client.get(`/api/v3/databoxes${dbid}/subdefs/`);
 
-        const subdefs: PhraseanetSubDef[] = [];
+        const subdefs: PhraseanetSubdefStruct[] = [];
 
         const dbxs = res.data.response.databoxes;
         Object.keys(dbxs).forEach(id => {

databox/indexer/src/handlers/phraseanet/shared.ts

@@ -1,5 +1,5 @@
 import {Asset} from '../../indexers';
-import {FieldMap, SubDef} from './types';
+import {FieldMap, PhraseanetSubdef} from './types';
 import { CPhraseanetRecord } from './CPhraseanetRecord';
 
 import {
@@ -36,7 +36,7 @@ export async function createAsset(
     tagIndex: TagIndex,
     shortcutIntoCollections: {id: string, path: string}[]
 ): Promise<Asset> {
-    const document: SubDef | undefined = record.subdefs.find(
+    const document: PhraseanetSubdef | undefined = record.subdefs.find(
         s => s.name === 'document'
     );
 

databox/indexer/src/handlers/phraseanet/types.d.ts

@@ -48,14 +48,6 @@ export type PhraseanetConfig = {
     databoxMapping: ConfigDataboxMapping[];
 };
 
-export type SubDef = {
-    name: string;
-    mime_type?: string;
-    permalink: {
-        url: string;
-    };
-};
-
 export enum PhraseanetMetadataType {
     Date = "date",
     Number = "number",
@@ -89,7 +81,7 @@ export type PhraseanetStatusBit = {
     state: boolean;
 };
 
-export type PhraseanetSubDef = {
+export type PhraseanetSubdefStruct = {
     type: string; // image | video | audio | document
     name: string; // thumbnail, thumbnail_gif, preview, preview_webm ...
     databox_id: number;
@@ -101,6 +93,21 @@ export type PhraseanetSubDef = {
     options: Record<string, any>;
 };
 
+export type PhraseanetSubdef = {
+    name: string;
+    height: number;
+    width: number;
+    filesize: number;
+    player_type: string;
+    mime_type: string;
+    created_on: string;
+    updated_on: string;
+    url: string;
+    permalink: {
+        url: string;
+    };
+};
+
 export type PhraseanetDatabox = {
     databox_id: string;
     name: string;
@@ -134,7 +141,10 @@ export type PhraseanetRecord = {
     uuid: string;
     title: string;
     original_name: string;
-    subdefs: SubDef[];
+    mime_type: string;
+    created_on: string;
+    updated_on: string;
+    subdefs: PhraseanetSubdef[];
     status: PhraseanetStatusBit[];
     metadata: PhraseanetMetadata[];
 };
@@ -148,7 +158,10 @@ export type PhraseanetStory = {
     uuid: string;
     title: string;
     original_name: string;
-    subdefs: SubDef[];
+    mime_type: string;
+    created_on: string;
+    updated_on: string;
+    subdefs: PhraseanetSubdef[];
     status: PhraseanetStatusBit[];
     metadata: PhraseanetMetadata[];
     children: PhraseanetRecord[];