Skip to content

Commit e359463

Browse files
brunozoricPavel910
brunozoric
and
Pavel910
authored
fix(db-dynamodb): tools for batch write (#4445)
Co-authored-by: Pavel Denisjuk <[email protected]>
1 parent edaf911 commit e359463

File tree

80 files changed

+2763
-2170
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+2763
-2170
lines changed

‎packages/api-elasticsearch-tasks/src/definitions/entry.ts

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
1-
import { Entity, TableDef } from "@webiny/db-dynamodb/toolbox";
1+
/**
2+
* TODO If adding GSIs to the Elasticsearch table, add them here.
3+
*/
4+
import type { TableDef } from "@webiny/db-dynamodb/toolbox";
5+
import type { IEntity } from "@webiny/db-dynamodb";
6+
import { createEntity } from "@webiny/db-dynamodb";
27

38
interface Params {
49
table: TableDef;
510
entityName: string;
611
}
712

8-
export const createEntry = (params: Params): Entity<any> => {
13+
export const createEntry = (params: Params): IEntity => {
914
const { table, entityName } = params;
10-
return new Entity({
15+
return createEntity({
1116
name: entityName,
1217
table,
1318
attributes: {
@@ -24,6 +29,9 @@ export const createEntry = (params: Params): Entity<any> => {
2429
},
2530
data: {
2631
type: "map"
32+
},
33+
TYPE: {
34+
type: "string"
2735
}
2836
}
2937
});

‎packages/api-elasticsearch-tasks/src/tasks/Manager.ts

Lines changed: 12 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,16 @@
11
import { DynamoDBDocument, getDocumentClient } from "@webiny/aws-sdk/client-dynamodb";
22
import { Client, createElasticsearchClient } from "@webiny/api-elasticsearch";
33
import { createTable } from "~/definitions";
4-
import { Context, IManager } from "~/types";
4+
import type { Context, IManager } from "~/types";
55
import { createEntry } from "~/definitions/entry";
6-
import { Entity } from "@webiny/db-dynamodb/toolbox";
7-
import { ITaskResponse } from "@webiny/tasks/response/abstractions";
8-
import { IIsCloseToTimeoutCallable, ITaskManagerStore } from "@webiny/tasks/runner/abstractions";
9-
import {
10-
batchReadAll,
11-
BatchReadItem,
12-
batchWriteAll,
13-
BatchWriteItem,
14-
BatchWriteResult
15-
} from "@webiny/db-dynamodb";
16-
import { ITimer } from "@webiny/handler-aws/utils";
6+
import type { ITaskResponse } from "@webiny/tasks/response/abstractions";
7+
import type {
8+
IIsCloseToTimeoutCallable,
9+
ITaskManagerStore
10+
} from "@webiny/tasks/runner/abstractions";
11+
import type { BatchReadItem, IEntity } from "@webiny/db-dynamodb";
12+
import { batchReadAll } from "@webiny/db-dynamodb";
13+
import type { ITimer } from "@webiny/handler-aws/utils";
1714

1815
export interface ManagerParams<T> {
1916
context: Context;
@@ -37,7 +34,7 @@ export class Manager<T> implements IManager<T> {
3734
public readonly store: ITaskManagerStore<T>;
3835
public readonly timer: ITimer;
3936

40-
private readonly entities: Record<string, Entity<any>> = {};
37+
private readonly entities: Record<string, IEntity> = {};
4138

4239
public constructor(params: ManagerParams<T>) {
4340
this.context = params.context;
@@ -64,7 +61,7 @@ export class Manager<T> implements IManager<T> {
6461
this.timer = params.timer;
6562
}
6663

67-
public getEntity(name: string): Entity<any> {
64+
public getEntity(name: string): IEntity {
6865
if (this.entities[name]) {
6966
return this.entities[name];
7067
}
@@ -75,17 +72,10 @@ export class Manager<T> implements IManager<T> {
7572
}));
7673
}
7774

78-
public async read<T>(items: BatchReadItem[]) {
75+
public async read<T>(items: BatchReadItem[]): Promise<T[]> {
7976
return await batchReadAll<T>({
8077
table: this.table,
8178
items
8279
});
8380
}
84-
85-
public async write(items: BatchWriteItem[]): Promise<BatchWriteResult> {
86-
return await batchWriteAll({
87-
table: this.table,
88-
items
89-
});
90-
}
9181
}

‎packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import {
66
} from "~/types";
77
import { ITaskResponse, ITaskResponseResult } from "@webiny/tasks/response/abstractions";
88
import { scan } from "~/helpers/scan";
9-
import { BatchWriteItem, ScanResponse } from "@webiny/db-dynamodb";
9+
import { createTableWriteBatch, ScanResponse } from "@webiny/db-dynamodb";
1010
import { IndexManager } from "~/settings";
1111
import { IIndexManager } from "~/settings/types";
1212

@@ -73,7 +73,10 @@ export class ReindexingTaskRunner {
7373
return this.response.done("No more items to process.");
7474
}
7575

76-
const batch: BatchWriteItem[] = [];
76+
const tableWriteBatch = createTableWriteBatch({
77+
table: this.manager.table
78+
});
79+
7780
for (const item of results.items) {
7881
/**
7982
* No index defined? Impossible but let's skip if really happens.
@@ -110,14 +113,13 @@ export class ReindexingTaskRunner {
110113
/**
111114
* Reindexing will be triggered by the `putBatch` method.
112115
*/
113-
batch.push(
114-
entity.putBatch({
115-
...item,
116-
modified: new Date().toISOString()
117-
})
118-
);
116+
tableWriteBatch.put(entity.entity, {
117+
...item,
118+
TYPE: item.TYPE || "unknown",
119+
modified: new Date().toISOString()
120+
});
119121
}
120-
await this.manager.write(batch);
122+
await tableWriteBatch.execute();
121123
/**
122124
* We always store the index settings, so we can restore them later.
123125
* Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.

‎packages/api-elasticsearch-tasks/src/types.ts

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
2-
import { Entity } from "@webiny/db-dynamodb/toolbox";
3-
import {
1+
import type { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
2+
import type {
43
Context as TasksContext,
54
IIsCloseToTimeoutCallable,
5+
ITaskManagerStore,
6+
ITaskResponse,
67
ITaskResponseDoneResultOutput
78
} from "@webiny/tasks/types";
8-
import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb";
9-
import { Client } from "@webiny/api-elasticsearch";
9+
import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb";
10+
import type { Client } from "@webiny/api-elasticsearch";
1011
import { createTable } from "~/definitions";
11-
import { ITaskResponse } from "@webiny/tasks/response/abstractions";
12-
import { ITaskManagerStore } from "@webiny/tasks/runner/abstractions";
13-
import { BatchWriteItem, BatchWriteResult } from "@webiny/db-dynamodb";
14-
import { ITimer } from "@webiny/handler-aws";
12+
import type { BatchReadItem, IEntity } from "@webiny/db-dynamodb";
13+
import type { ITimer } from "@webiny/handler-aws";
14+
import type { GenericRecord } from "@webiny/api/types";
1515

1616
export interface Context extends ElasticsearchContext, TasksContext {}
1717

@@ -42,17 +42,18 @@ export interface IElasticsearchIndexingTaskValues {
4242
}
4343

4444
export interface AugmentedError extends Error {
45-
data?: Record<string, any>;
45+
data?: GenericRecord;
4646
[key: string]: any;
4747
}
4848

4949
export interface IDynamoDbElasticsearchRecord {
5050
PK: string;
5151
SK: string;
52+
TYPE?: string;
5253
index: string;
5354
_et?: string;
5455
entity: string;
55-
data: Record<string, any>;
56+
data: GenericRecord;
5657
modified: string;
5758
}
5859

@@ -70,7 +71,7 @@ export interface IManager<
7071
readonly store: ITaskManagerStore<T>;
7172
readonly timer: ITimer;
7273

73-
getEntity: (name: string) => Entity<any>;
74+
getEntity: (name: string) => IEntity;
7475

75-
write: (items: BatchWriteItem[]) => Promise<BatchWriteResult>;
76+
read<T>(items: BatchReadItem[]): Promise<T[]>;
7677
}

‎packages/api-file-manager-ddb/src/operations/AliasesStorageOperations.ts

Lines changed: 24 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
1-
import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb";
2-
import { Entity, Table } from "@webiny/db-dynamodb/toolbox";
3-
import {
4-
FileManagerAliasesStorageOperations,
1+
import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb";
2+
import type { Entity, Table } from "@webiny/db-dynamodb/toolbox";
3+
import type {
54
File,
6-
FileAlias
5+
FileAlias,
6+
FileManagerAliasesStorageOperations
77
} from "@webiny/api-file-manager/types";
88
import {
9-
BatchWriteItem,
10-
batchWriteAll,
9+
createEntityWriteBatch,
1110
createStandardEntity,
1211
createTable,
1312
DbItem,
@@ -39,52 +38,49 @@ export class AliasesStorageOperations implements FileManagerAliasesStorageOperat
3938

4039
async deleteAliases(file: File): Promise<void> {
4140
const aliasItems = await this.getExistingAliases(file);
42-
const items: BatchWriteItem[] = [];
4341

44-
aliasItems.forEach(item => {
45-
items.push(
46-
this.aliasEntity.deleteBatch({
42+
const batchWrite = createEntityWriteBatch({
43+
entity: this.aliasEntity,
44+
delete: aliasItems.map(item => {
45+
return {
4746
PK: this.createPartitionKey({
4847
id: item.fileId,
4948
tenant: item.tenant,
5049
locale: item.locale
5150
}),
5251
SK: `ALIAS#${item.alias}`
53-
})
54-
);
52+
};
53+
})
5554
});
5655

57-
await batchWriteAll({ table: this.table, items });
56+
await batchWrite.execute();
5857
}
5958

6059
async storeAliases(file: File): Promise<void> {
61-
const items: BatchWriteItem[] = [];
6260
const existingAliases = await this.getExistingAliases(file);
6361
const newAliases = this.createNewAliasesRecords(file, existingAliases);
6462

65-
newAliases.forEach(alias => {
66-
items.push(this.aliasEntity.putBatch(alias));
63+
const batchWrite = createEntityWriteBatch({
64+
entity: this.aliasEntity
6765
});
66+
for (const alias of newAliases) {
67+
batchWrite.put(alias);
68+
}
6869

6970
// Delete aliases that are in the DB but are NOT in the file.
7071
for (const data of existingAliases) {
7172
if (!file.aliases.some(alias => data.alias === alias)) {
72-
items.push(
73-
this.aliasEntity.deleteBatch({
74-
PK: this.createPartitionKey(file),
75-
SK: `ALIAS#${data.alias}`
76-
})
77-
);
73+
batchWrite.delete({
74+
PK: this.createPartitionKey(file),
75+
SK: `ALIAS#${data.alias}`
76+
});
7877
}
7978
}
8079

81-
await batchWriteAll({
82-
table: this.table,
83-
items
84-
});
80+
await batchWrite.execute();
8581
}
8682

87-
private async getExistingAliases(file: File) {
83+
private async getExistingAliases(file: File): Promise<FileAlias[]> {
8884
const aliases = await queryAll<{ data: FileAlias }>({
8985
entity: this.aliasEntity,
9086
partitionKey: this.createPartitionKey(file),

‎packages/api-form-builder-so-ddb-es/src/definitions/elasticsearch.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ export const createElasticsearchEntity = (params: Params) => {
2828
TYPE: {
2929
type: "string"
3030
},
31-
3231
...(attributes || {})
3332
}
3433
});

0 commit comments

Comments
 (0)