This is an automated email from the ASF dual-hosted git repository.
linxinyuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/texera.git
The following commit(s) were added to refs/heads/master by this push:
new f4727a6209 feat: enable configurable multipart upload settings using
database (#3622)
f4727a6209 is described below
commit f4727a6209bfb132debe2ee6a02b10fdb2935f6a
Author: Xuan Gu <[email protected]>
AuthorDate: Fri Aug 8 19:48:59 2025 -0700
feat: enable configurable multipart upload settings using database (#3622)
### **Purpose**
This PR extends the [configuration-CRUD
framework](https://github.com/Texera/texera/issues/3528) in Admin
Settings to include multipart-upload parameters. Admins can adjust
global defaults for max file upload size, chunk size, and concurrent
upload count directly from Admin → General Settings → Dataset. Changes
take effect immediately without service restarts and persist in the
_site_settings_ table.
### **Changes**
- admin-settings.component.ts/html/scss: Added a Dataset card with three
controls (max file size, chunk size, concurrency).
- files-uploader.component.ts: Switched to reading the
“single-file-upload-maximum-size-mb” setting from the database.
- dataset-detail.component.ts/dataset.service.ts: Extended
multipartUpload() signature to accept partSizeMB and concurrencyLimit
and forward them to the backend.
- default.conf: Defined upload default values.
-
gui.conf/gui-config.service.ts/gui-config.service.mock.ts/GuiConfig.scala/ConfigResource.scala:
Migrated upload-setting storage to the new keys and removed the original
byte-based fields.
- app.module.ts: Imported NzInputNumberModule to enable number-input
controls in the admin UI.
### **Demonstration**
Settings Interface:
<img width="788" height="361" alt="Uploading"
src="https://github.com/user-attachments/assets/81fbaff7-dad0-4132-9454-2b07854dc280";
/>
---------
Co-authored-by: Xinyuan Lin <[email protected]>
---
.../texera/service/resource/ConfigResource.scala | 3 --
core/config/src/main/resources/default.conf | 12 +++++
core/config/src/main/resources/gui.conf | 15 ------
.../edu/uci/ics/texera/config/GuiConfig.scala | 8 ----
core/gui/src/app/app.module.ts | 2 +
.../app/common/service/gui-config.service.mock.ts | 3 --
core/gui/src/app/common/type/gui-config.ts | 3 --
.../admin/settings/admin-settings.component.html | 49 +++++++++++++++++++
.../admin/settings/admin-settings.component.scss | 14 ++++++
.../admin/settings/admin-settings.component.ts | 55 ++++++++++++++++++++++
.../files-uploader/files-uploader.component.ts | 20 +++++---
.../dataset-detail.component.ts | 30 ++++++++++--
.../service/user/dataset/dataset.service.ts | 10 ++--
13 files changed, 179 insertions(+), 45 deletions(-)
diff --git
a/core/config-service/src/main/scala/edu/uci/ics/texera/service/resource/ConfigResource.scala
b/core/config-service/src/main/scala/edu/uci/ics/texera/service/resource/ConfigResource.scala
index df1feaf7fb..23880f3c08 100644
---
a/core/config-service/src/main/scala/edu/uci/ics/texera/service/resource/ConfigResource.scala
+++
b/core/config-service/src/main/scala/edu/uci/ics/texera/service/resource/ConfigResource.scala
@@ -45,9 +45,6 @@ class ConfigResource {
"asyncRenderingEnabled" ->
GuiConfig.guiWorkflowWorkspaceAsyncRenderingEnabled,
"timetravelEnabled" -> GuiConfig.guiWorkflowWorkspaceTimetravelEnabled,
"productionSharedEditingServer" ->
GuiConfig.guiWorkflowWorkspaceProductionSharedEditingServer,
- "singleFileUploadMaximumSizeMB" ->
GuiConfig.guiDatasetSingleFileUploadMaximumSizeMB,
- "maxNumberOfConcurrentUploadingFileChunks" ->
GuiConfig.guiDatasetMaxNumberOfConcurrentUploadingFileChunks,
- "multipartUploadChunkSizeByte" ->
GuiConfig.guiDatasetMultipartUploadChunkSizeByte,
"defaultDataTransferBatchSize" ->
GuiConfig.guiWorkflowWorkspaceDefaultDataTransferBatchSize,
"workflowEmailNotificationEnabled" ->
GuiConfig.guiWorkflowWorkspaceWorkflowEmailNotificationEnabled,
"sharingComputingUnitEnabled" ->
ComputingUnitConfig.sharingComputingUnitEnabled,
diff --git a/core/config/src/main/resources/default.conf
b/core/config/src/main/resources/default.conf
index fe60ed0f35..fa6e949c1d 100644
--- a/core/config/src/main/resources/default.conf
+++ b/core/config/src/main/resources/default.conf
@@ -50,3 +50,15 @@ gui {
about_enabled = true
}
}
+
+dataset {
+ # the file size limit for dataset upload
+ single_file_upload_max_size_mb = 20
+
+ # the maximum number of file chunks that can be held in the memory;
+ # you may increase this number if your deployment environment has enough
memory resource
+ max_number_of_concurrent_uploading_file_chunks = 10
+
+ # the size of each chunk during the multipart upload of file
+ multipart_upload_chunk_size_mb = 50
+}
diff --git a/core/config/src/main/resources/gui.conf
b/core/config/src/main/resources/gui.conf
index ebdc4f2807..14d3c69403 100644
--- a/core/config/src/main/resources/gui.conf
+++ b/core/config/src/main/resources/gui.conf
@@ -101,19 +101,4 @@ gui {
workflow-email-notification-enabled = false
workflow-email-notification-enabled =
${?GUI_WORKFLOW_WORKSPACE_WORKFLOW_EMAIL_NOTIFICATION_ENABLED}
}
-
- dataset {
- # the file size limit for dataset upload
- single-file-upload-maximum-size-mb = 20
- single-file-upload-maximum-size-mb =
${?GUI_DATASET_SINGLE_FILE_UPLOAD_MAXIMUM_SIZE_MB}
-
- # the maximum number of file chunks that can be held in the memory;
- # you may increase this number if your deployment environment has enough
memory resource.
- max-number-of-concurrent-uploading-file-chunks = 10
- max-number-of-concurrent-uploading-file-chunks =
${?GUI_DATASET_MAX_NUMBER_OF_CONCURRENT_UPLOADING_FILE_CHUNKS}
-
- # the size of each chunk during the multipart upload of file
- multipart-upload-chunk-size-byte = 52428800 # 50 MB
- multipart-upload-chunk-size-byte =
${?GUI_DATASET_MULTIPART_UPLOAD_CHUNK_SIZE_BYTE}
- }
}
\ No newline at end of file
diff --git
a/core/config/src/main/scala/edu/uci/ics/texera/config/GuiConfig.scala
b/core/config/src/main/scala/edu/uci/ics/texera/config/GuiConfig.scala
index 4ffe495747..b64edcfe41 100644
--- a/core/config/src/main/scala/edu/uci/ics/texera/config/GuiConfig.scala
+++ b/core/config/src/main/scala/edu/uci/ics/texera/config/GuiConfig.scala
@@ -65,12 +65,4 @@ object GuiConfig {
conf.getInt("gui.workflow-workspace.operator-console-message-buffer-size")
val guiWorkflowWorkspaceWorkflowEmailNotificationEnabled: Boolean =
conf.getBoolean("gui.workflow-workspace.workflow-email-notification-enabled")
-
- // GUI Dataset Configuration
- val guiDatasetSingleFileUploadMaximumSizeMB: Int =
- conf.getInt("gui.dataset.single-file-upload-maximum-size-mb")
- val guiDatasetMaxNumberOfConcurrentUploadingFileChunks: Int =
- conf.getInt("gui.dataset.max-number-of-concurrent-uploading-file-chunks")
- val guiDatasetMultipartUploadChunkSizeByte: Long =
- conf.getLong("gui.dataset.multipart-upload-chunk-size-byte")
}
diff --git a/core/gui/src/app/app.module.ts b/core/gui/src/app/app.module.ts
index f7f228a62e..13759e6e55 100644
--- a/core/gui/src/app/app.module.ts
+++ b/core/gui/src/app/app.module.ts
@@ -171,6 +171,7 @@ import { NzSliderModule } from "ng-zorro-antd/slider";
import { AdminSettingsComponent } from
"./dashboard/component/admin/settings/admin-settings.component";
import { catchError, of } from "rxjs";
import { FormlyRepeatDndComponent } from
"./common/formly/repeat-dnd/repeat-dnd.component";
+import { NzInputNumberModule } from "ng-zorro-antd/input-number";
registerLocaleData(en);
@@ -328,6 +329,7 @@ registerLocaleData(en);
NzEmptyModule,
NzDividerModule,
NzProgressModule,
+ NzInputNumberModule,
],
providers: [
provideNzI18n(en_US),
diff --git a/core/gui/src/app/common/service/gui-config.service.mock.ts
b/core/gui/src/app/common/service/gui-config.service.mock.ts
index 69b3b61bbb..60f7e7413c 100644
--- a/core/gui/src/app/common/service/gui-config.service.mock.ts
+++ b/core/gui/src/app/common/service/gui-config.service.mock.ts
@@ -42,9 +42,6 @@ export class MockGuiConfigService {
timetravelEnabled: false,
productionSharedEditingServer: false,
pythonLanguageServerPort: "3000",
- singleFileUploadMaximumSizeMB: 100,
- maxNumberOfConcurrentUploadingFileChunks: 5,
- multipartUploadChunkSizeByte: 1048576, // 1MB
defaultDataTransferBatchSize: 100,
workflowEmailNotificationEnabled: false,
sharingComputingUnitEnabled: false,
diff --git a/core/gui/src/app/common/type/gui-config.ts
b/core/gui/src/app/common/type/gui-config.ts
index 4da5d98131..d60ba1cf01 100644
--- a/core/gui/src/app/common/type/gui-config.ts
+++ b/core/gui/src/app/common/type/gui-config.ts
@@ -33,9 +33,6 @@ export interface GuiConfig {
timetravelEnabled: boolean;
productionSharedEditingServer: boolean;
pythonLanguageServerPort: string;
- singleFileUploadMaximumSizeMB: number;
- maxNumberOfConcurrentUploadingFileChunks: number;
- multipartUploadChunkSizeByte: number;
defaultDataTransferBatchSize: number;
workflowEmailNotificationEnabled: boolean;
sharingComputingUnitEnabled: boolean;
diff --git
a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html
b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html
index 0951a81b26..a158389102 100644
---
a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html
+++
b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html
@@ -238,3 +238,52 @@
</div>
</div>
</nz-card>
+
+<nz-card nzTitle="Dataset">
+ <div class="settings-row">
+ <span>File Size:</span>
+ <nz-input-number
+ [(ngModel)]="maxFileSizeMB"
+ [nzMin]="1"
+ [nzStep]="10">
+ </nz-input-number>
+ </div>
+ <div class="help-text-number">Maximum size allowed for individual file
uploads (MB).</div>
+
+ <div class="settings-row">
+ <span>Concurrent Parts:</span>
+ <nz-input-number
+ [(ngModel)]="maxConcurrentChunks"
+ [nzMin]="1"
+ [nzStep]="1">
+ </nz-input-number>
+ </div>
+ <div class="help-text-number">
+ Number of file chunks that can be uploaded simultaneously. Higher values
use more memory.
+ </div>
+
+ <div class="settings-row">
+ <span>Part Size:</span>
+ <nz-input-number
+ [(ngModel)]="chunkSizeMB"
+ [nzMin]="1"
+ [nzStep]="10">
+ </nz-input-number>
+ </div>
+ <div class="help-text-number">Size of each chunk during multipart upload in
MB. Larger chunks use more memory.</div>
+
+ <div class="button-row">
+ <button
+ nz-button
+ nzType="primary"
+ (click)="saveDatasetSettings()">
+ Save
+ </button>
+ <button
+ nz-button
+ nzType="default"
+ (click)="resetDatasetSettings()">
+ Reset
+ </button>
+ </div>
+</nz-card>
diff --git
a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.scss
b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.scss
index ae34fe6405..073d6352aa 100644
---
a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.scss
+++
b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.scss
@@ -77,3 +77,17 @@ details[open] .arrow {
margin-left: 2rem;
margin-bottom: 8px;
}
+
+.settings-row {
+ display: grid;
+ grid-template-columns: 130px 320px;
+ column-gap: 6px;
+ margin-bottom: 4px;
+}
+
+.help-text-number {
+ margin-left: 140px;
+ font-size: 12px;
+ color: #888;
+ margin-bottom: 20px;
+}
diff --git
a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts
b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts
index 773dd9474f..9633bb19b6 100644
---
a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts
+++
b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts
@@ -22,6 +22,7 @@ import { AdminSettingsService } from
"../../../service/admin/settings/admin-sett
import { NzMessageService } from "ng-zorro-antd/message";
import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy";
import { SidebarTabs } from "../../../../common/type/gui-config";
+import { forkJoin } from "rxjs";
@UntilDestroy()
@Component({
@@ -47,12 +48,17 @@ export class AdminSettingsComponent implements OnInit {
about_enabled: false,
};
+ maxFileSizeMB: number = 20;
+ maxConcurrentChunks: number = 10;
+ chunkSizeMB: number = 50;
+
constructor(
private adminSettingsService: AdminSettingsService,
private message: NzMessageService
) {}
ngOnInit(): void {
this.loadTabs();
+ this.loadDatasetSetting();
}
private loadTabs(): void {
@@ -64,6 +70,21 @@ export class AdminSettingsComponent implements OnInit {
});
}
+ private loadDatasetSetting(): void {
+ this.adminSettingsService
+ .getSetting("single_file_upload_max_size_mb")
+ .pipe(untilDestroyed(this))
+ .subscribe(value => (this.maxFileSizeMB = parseInt(value)));
+ this.adminSettingsService
+ .getSetting("max_number_of_concurrent_uploading_file_chunks")
+ .pipe(untilDestroyed(this))
+ .subscribe(value => (this.maxConcurrentChunks = parseInt(value)));
+ this.adminSettingsService
+ .getSetting("multipart_upload_chunk_size_mb")
+ .pipe(untilDestroyed(this))
+ .subscribe(value => (this.chunkSizeMB = parseInt(value)));
+ }
+
onFileChange(type: "logo" | "mini_logo" | "favicon", event: Event): void {
const file = (event.target as HTMLInputElement).files?.[0];
if (file && file.type.startsWith("image/")) {
@@ -181,4 +202,38 @@ export class AdminSettingsComponent implements OnInit {
this.message.info("Resetting tabs...");
setTimeout(() => window.location.reload(), 500);
}
+
+ saveDatasetSettings(): void {
+ if (this.maxFileSizeMB < 1 || this.maxConcurrentChunks < 1 ||
this.chunkSizeMB < 1) {
+ this.message.info("Value must be at least 1.");
+ return;
+ }
+
+ const saveRequests = [
+
this.adminSettingsService.updateSetting("single_file_upload_max_size_mb",
this.maxFileSizeMB.toString()),
+ this.adminSettingsService.updateSetting(
+ "max_number_of_concurrent_uploading_file_chunks",
+ this.maxConcurrentChunks.toString()
+ ),
+
this.adminSettingsService.updateSetting("multipart_upload_chunk_size_mb",
this.chunkSizeMB.toString()),
+ ];
+
+ forkJoin(saveRequests)
+ .pipe(untilDestroyed(this))
+ .subscribe({
+ next: () => this.message.success("Dataset upload settings saved
successfully."),
+ error: () => this.message.error("Failed to save dataset settings."),
+ });
+ }
+
+ resetDatasetSettings(): void {
+ [
+ "single_file_upload_max_size_mb",
+ "max_number_of_concurrent_uploading_file_chunks",
+ "multipart_upload_chunk_size_mb",
+ ].forEach(setting =>
this.adminSettingsService.resetSetting(setting).pipe(untilDestroyed(this)).subscribe({}));
+
+ this.message.info("Resetting dataset settings...");
+ setTimeout(() => window.location.reload(), 500);
+ }
}
diff --git
a/core/gui/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts
b/core/gui/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts
index eca4f203f7..c9bec24c60 100644
---
a/core/gui/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts
+++
b/core/gui/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts
@@ -17,7 +17,7 @@
* under the License.
*/
-import { Component, EventEmitter, Input, Output } from "@angular/core";
+import { Component, EventEmitter, Input, OnInit, Output } from "@angular/core";
import { FileUploadItem } from "../../../type/dashboard-file.interface";
import { NgxFileDropEntry } from "ngx-file-drop";
import {
@@ -26,8 +26,10 @@ import {
getPathsUnderOrEqualDatasetFileNode,
} from "../../../../common/type/datasetVersionFileTree";
import { NotificationService } from
"../../../../common/service/notification/notification.service";
-import { GuiConfigService } from
"../../../../common/service/gui-config.service";
+import { AdminSettingsService } from
"../../../service/admin/settings/admin-settings.service";
+import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy";
+@UntilDestroy()
@Component({
selector: "texera-user-files-uploader",
templateUrl: "./files-uploader.component.html",
@@ -46,11 +48,17 @@ export class FilesUploaderComponent {
// four types: "success", "info", "warning" and "error"
fileUploadBannerType: "error" | "success" | "info" | "warning" = "success";
fileUploadBannerMessage: string = "";
+ singleFileUploadMaxSizeMB: number = 20;
constructor(
private notificationService: NotificationService,
- private config: GuiConfigService
- ) {}
+ private adminSettingsService: AdminSettingsService
+ ) {
+ this.adminSettingsService
+ .getSetting("single_file_upload_max_size_mb")
+ .pipe(untilDestroyed(this))
+ .subscribe(value => (this.singleFileUploadMaxSizeMB = parseInt(value)));
+ }
hideBanner() {
this.fileUploadingFinished = false;
@@ -71,10 +79,10 @@ export class FilesUploaderComponent {
const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
fileEntry.file(file => {
// Check the file size here
- if (file.size > this.config.env.singleFileUploadMaximumSizeMB *
1024 * 1024) {
+ if (file.size > this.singleFileUploadMaxSizeMB * 1024 * 1024) {
// If the file is too large, reject the promise
this.notificationService.error(
- `File ${file.name}'s size exceeds the maximum limit of
${this.config.env.singleFileUploadMaximumSizeMB}MB.`
+ `File ${file.name}'s size exceeds the maximum limit of
${this.singleFileUploadMaxSizeMB}MB.`
);
reject(null);
} else {
diff --git
a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts
b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts
index 317604dfda..9dd4de0c40 100644
---
a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts
+++
b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts
@@ -39,6 +39,7 @@ import { FileUploadItem } from
"../../../../type/dashboard-file.interface";
import { DatasetStagedObject } from
"../../../../../common/type/dataset-staged-object";
import { NzModalService } from "ng-zorro-antd/modal";
import { UserDatasetVersionCreatorComponent } from
"./user-dataset-version-creator/user-dataset-version-creator.component";
+import { AdminSettingsService } from
"../../../../service/admin/settings/admin-settings.service";
export const THROTTLE_TIME_MS = 1000;
@@ -76,6 +77,9 @@ export class DatasetDetailComponent implements OnInit {
public displayPreciseViewCount = false;
userHasPendingChanges: boolean = false;
+ // Uploading setting
+ chunkSizeMB: number = 50;
+ maxConcurrentChunks: number = 10;
// List of upload tasks – each task tracked by its filePath
public uploadTasks: Array<
@@ -94,7 +98,8 @@ export class DatasetDetailComponent implements OnInit {
private notificationService: NotificationService,
private downloadService: DownloadService,
private userService: UserService,
- private hubService: HubService
+ private hubService: HubService,
+ private adminSettingsService: AdminSettingsService
) {
this.userService
.userChanged()
@@ -159,8 +164,9 @@ export class DatasetDetailComponent implements OnInit {
.subscribe((isLiked: LikedStatus[]) => {
this.isLiked = isLiked.length > 0 ? isLiked[0].isLiked : false;
});
- }
+ this.loadUploadSettings();
+ }
public onClickOpenVersionCreator() {
if (this.did) {
const modal = this.modalService.create({
@@ -309,6 +315,17 @@ export class DatasetDetailComponent implements OnInit {
return task.filePath;
}
+ private loadUploadSettings(): void {
+ this.adminSettingsService
+ .getSetting("multipart_upload_chunk_size_mb")
+ .pipe(untilDestroyed(this))
+ .subscribe(value => (this.chunkSizeMB = parseInt(value)));
+ this.adminSettingsService
+ .getSetting("max_number_of_concurrent_uploading_file_chunks")
+ .pipe(untilDestroyed(this))
+ .subscribe(value => (this.maxConcurrentChunks = parseInt(value)));
+ }
+
onNewUploadFilesChanged(files: FileUploadItem[]) {
if (this.did) {
files.forEach((file, idx) => {
@@ -320,10 +337,15 @@ export class DatasetDetailComponent implements OnInit {
uploadId: "",
physicalAddress: "",
});
-
// Start multipart upload
this.datasetService
- .multipartUpload(this.datasetName, file.name, file.file)
+ .multipartUpload(
+ this.datasetName,
+ file.name,
+ file.file,
+ this.chunkSizeMB * 1024 * 1024,
+ this.maxConcurrentChunks
+ )
.pipe(untilDestroyed(this))
.subscribe({
next: progress => {
diff --git a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts
b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts
index 29bc627cd8..226b1edb97 100644
--- a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts
+++ b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts
@@ -137,10 +137,14 @@ export class DatasetService {
* Handles multipart upload for large files using RxJS,
* with a concurrency limit on how many parts we process in parallel.
*/
- public multipartUpload(datasetName: string, filePath: string, file: File):
Observable<MultipartUploadProgress> {
- const partSize = this.config.env.multipartUploadChunkSizeByte;
+ public multipartUpload(
+ datasetName: string,
+ filePath: string,
+ file: File,
+ partSize: number,
+ concurrencyLimit: number
+ ): Observable<MultipartUploadProgress> {
const partCount = Math.ceil(file.size / partSize);
- const concurrencyLimit =
this.config.env.maxNumberOfConcurrentUploadingFileChunks;
// track progress bar
let totalBytesUploaded = 0;
