This is an automated email from the ASF dual-hosted git repository. linxinyuan pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/texera.git
commit f4827796d109f3702f8777b93ee161edfd914949 Author: Xinyuan Lin <[email protected]> AuthorDate: Tue Sep 30 14:32:07 2025 -0700 Merge main to asf-site (#3791) Signed-off-by: Ma77Ball <[email protected]> Signed-off-by: PJ Fanning <[email protected]> Co-authored-by: ali risheh <[email protected]> Co-authored-by: Meng Wang <[email protected]> Co-authored-by: yunyad <[email protected]> Co-authored-by: Xuan Gu <[email protected]> Co-authored-by: colinthebomb1 <[email protected]> Co-authored-by: Ma77Ball <[email protected]> Co-authored-by: Copilot <[email protected]> Co-authored-by: PJ Fanning <[email protected]> Co-authored-by: Seongjin Yoon <[email protected]> Co-authored-by: Seongjin Yoon <[email protected]> Co-authored-by: Seongjin Yoon <[email protected]> Co-authored-by: Grace Chia <[email protected]> --- .asf.yaml | 3 + core/access-control-service/build.sbt | 81 ++++++++ .../project/build.properties | 18 ++ .../access-control-service-web-config.yaml | 33 ++++ .../src/main/resources/logback.xml | 55 ++++++ .../ics/texera/service/AccessControlService.scala | 78 ++++++++ .../AccessControlServiceConfiguration.scala | 22 +++ .../service/resource/AccessControlResource.scala | 132 +++++++++++++ .../service/resource/HealthCheckResource.scala | 30 +++ .../uci/ics/texera/AccessControlResourceSpec.scala | 220 +++++++++++++++++++++ .../ics/texera/auth/util/ComputingUnitAccess.scala | 55 ++++++ .../edu/uci/ics/texera/auth/util/HeaderField.scala | 27 +++ core/build.sbt | 10 + core/config/src/main/resources/default.conf | 5 +- .../scala/edu/uci/ics/amber/util/PathUtils.scala | 2 + .../admin/settings/admin-settings.component.html | 12 ++ .../admin/settings/admin-settings.component.ts | 17 +- .../dataset-detail.component.html | 118 +++++++---- .../dataset-detail.component.scss | 19 +- .../dataset-detail.component.ts | 216 +++++++++++++------- ...user-dataset-staged-objects-list.component.html | 1 - .../user/user-quota/user-quota.component.html | 23 +-- .../user/user-quota/user-quota.component.scss | 4 +- .../service/user/download/download.service.ts | 4 +- .../left-panel/settings/settings.component.ts | 6 + .../computing-unit-selection.component.ts | 2 +- .../workflow-editor/workflow-editor.component.ts | 134 +++++++++++-- .../computing-unit-status.service.ts | 7 +- .../workspace/service/joint-ui/joint-ui.service.ts | 38 ---- .../SklearnTrainingAdaptiveBoosting.png | Bin 0 -> 117082 bytes .../operator_images/SklearnTrainingBagging.png | Bin 0 -> 60221 bytes .../SklearnTrainingBernoulliNaiveBayes.png | Bin 0 -> 433434 bytes .../SklearnTrainingComplementNaiveBayes.png | Bin 0 -> 74896 bytes .../SklearnTrainingDecisionTree.png | Bin 0 -> 7095 bytes .../operator_images/SklearnTrainingDummy.png | Bin 0 -> 39008 bytes .../operator_images/SklearnTrainingExtraTree.png | Bin 0 -> 20903 bytes .../operator_images/SklearnTrainingExtraTrees.png | Bin 0 -> 75482 bytes .../SklearnTrainingGaussianNaiveBayes.png | Bin 0 -> 69880 bytes .../SklearnTrainingGradientBoosting.png | Bin 0 -> 100542 bytes .../assets/operator_images/SklearnTrainingKNN.png | Bin 0 -> 96537 bytes .../SklearnTrainingLinearRegression.png | Bin 0 -> 13177 bytes .../operator_images/SklearnTrainingLinearSVM.png | Bin 0 -> 17599 bytes .../SklearnTrainingLogisticRegression.png | Bin 0 -> 18324 bytes .../SklearnTrainingLogisticRegressionCV.png | Bin 0 -> 10842 bytes .../SklearnTrainingMultiLayerPerceptron.png | Bin 0 -> 128735 bytes .../SklearnTrainingMultinomialNaiveBayes.png | Bin 0 -> 34729 bytes .../SklearnTrainingNearestCentroid.png | Bin 0 -> 214245 bytes .../SklearnTrainingPassiveAggressive.png | Bin 0 -> 9322 bytes .../operator_images/SklearnTrainingPerceptron.png | Bin 0 -> 13079 bytes .../SklearnTrainingProbabilityCalibration.png | Bin 0 -> 83338 bytes .../SklearnTrainingRandomForest.png | Bin 0 -> 81937 bytes .../operator_images/SklearnTrainingRidge.png | Bin 0 -> 24635 bytes .../operator_images/SklearnTrainingRidgeCV.png | Bin 0 -> 16258 bytes .../assets/operator_images/SklearnTrainingSDG.png | Bin 0 -> 22220 bytes .../assets/operator_images/SklearnTrainingSVM.png | Bin 0 -> 17776 bytes .../edu/uci/ics/amber/operator/LogicalOp.scala | 67 ++++--- .../SklearnTrainingLinearRegressionOpDesc.scala | 25 +++ 57 files changed, 1251 insertions(+), 213 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index f8028b19a8..8bd6ce570b 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -78,3 +78,6 @@ notifications: pullrequests: [email protected] discussions: [email protected] jobs: [email protected] + +publish: + whoami: asf-site diff --git a/core/access-control-service/build.sbt b/core/access-control-service/build.sbt new file mode 100644 index 0000000000..052dad4c13 --- /dev/null +++ b/core/access-control-service/build.sbt @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import scala.collection.Seq + +name := "access-control-service" +organization := "edu.uci.ics" +version := "1.0.0" + +scalaVersion := "2.13.12" + +enablePlugins(JavaAppPackaging) + +// Enable semanticdb for Scalafix +ThisBuild / semanticdbEnabled := true +ThisBuild / semanticdbVersion := scalafixSemanticdb.revision + +// Manage dependency conflicts by always using the latest revision +ThisBuild / conflictManager := ConflictManager.latestRevision + +// Restrict parallel execution of tests to avoid conflicts +Global / concurrentRestrictions += Tags.limit(Tags.Test, 1) + +///////////////////////////////////////////////////////////////////////////// +// Compiler Options +///////////////////////////////////////////////////////////////////////////// + +// Scala compiler options +Compile / scalacOptions ++= Seq( + "-Xelide-below", "WARNING", // Turn on optimizations with "WARNING" as the threshold + "-feature", // Check feature warnings + "-deprecation", // Check deprecation warnings + "-Ywarn-unused:imports" // Check for unused imports +) + +///////////////////////////////////////////////////////////////////////////// +// Version Variables +///////////////////////////////////////////////////////////////////////////// + +val dropwizardVersion = "4.0.7" +val mockitoVersion = "5.4.0" +val assertjVersion = "3.24.2" + +///////////////////////////////////////////////////////////////////////////// +// Test-related Dependencies +///////////////////////////////////////////////////////////////////////////// + +libraryDependencies ++= Seq( + "org.scalamock" %% "scalamock" % "5.2.0" % Test, // ScalaMock + "org.scalatest" %% "scalatest" % "3.2.17" % Test, // ScalaTest + "io.dropwizard" % "dropwizard-testing" % dropwizardVersion % Test, // Dropwizard Testing + "org.mockito" % "mockito-core" % mockitoVersion % Test, // Mockito for mocking + "org.assertj" % "assertj-core" % assertjVersion % Test, // AssertJ for assertions + "com.novocode" % "junit-interface" % "0.11" % Test // SBT interface for JUnit +) + +///////////////////////////////////////////////////////////////////////////// +// Dependencies +///////////////////////////////////////////////////////////////////////////// + +// Core Dependencies +libraryDependencies ++= Seq( + "io.dropwizard" % "dropwizard-core" % dropwizardVersion, + "io.dropwizard" % "dropwizard-auth" % dropwizardVersion, // Dropwizard Authentication module + "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.2", + "org.playframework" %% "play-json" % "3.1.0-M1", +) \ No newline at end of file diff --git a/core/access-control-service/project/build.properties b/core/access-control-service/project/build.properties new file mode 100644 index 0000000000..5a15dd8541 --- /dev/null +++ b/core/access-control-service/project/build.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +sbt.version = 1.9.9 \ No newline at end of file diff --git a/core/access-control-service/src/main/resources/access-control-service-web-config.yaml b/core/access-control-service/src/main/resources/access-control-service-web-config.yaml new file mode 100644 index 0000000000..e8d17cec28 --- /dev/null +++ b/core/access-control-service/src/main/resources/access-control-service-web-config.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +server: + applicationConnectors: + - type: http + port: 9096 + adminConnectors: [] + +logging: + level: INFO + appenders: + - type: console + threshold: INFO + - type: file + currentLogFilename: logs/access-control-service.log + archive: true + archivedLogFilenamePattern: logs/access-control-service-%d.log.gz + archivedFileCount: 5 \ No newline at end of file diff --git a/core/access-control-service/src/main/resources/logback.xml b/core/access-control-service/src/main/resources/logback.xml new file mode 100644 index 0000000000..4763107b50 --- /dev/null +++ b/core/access-control-service/src/main/resources/logback.xml @@ -0,0 +1,55 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +<configuration> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder + by default --> + <encoder> + <pattern>[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n + </pattern> + </encoder> + </appender> + + + <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>../log/access-control-service.log</file> + <immediateFlush>true</immediateFlush> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>../log/access-control-service-%d{yyyy-MM-dd}.log.gz</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n</pattern> + </encoder> + </appender> + + <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>8192</queueSize> + <neverBlock>true</neverBlock> + <appender-ref ref="FILE"/> + </appender> + + <root level="INFO"> + <appender-ref ref="ASYNC"/> + <appender-ref ref="STDOUT"/> + </root> + <logger name="org.apache" level="WARN"/> + <logger name="httpclient" level="WARN"/> + <logger name="io.grpc.netty" level="WARN"/> +</configuration> \ No newline at end of file diff --git a/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/AccessControlService.scala b/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/AccessControlService.scala new file mode 100644 index 0000000000..02278fd97a --- /dev/null +++ b/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/AccessControlService.scala @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package edu.uci.ics.texera.service + +import io.dropwizard.core.Application +import io.dropwizard.core.setup.{Bootstrap, Environment} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.typesafe.scalalogging.LazyLogging +import edu.uci.ics.amber.config.StorageConfig +import edu.uci.ics.amber.util.PathUtils.{configServicePath, accessControlServicePath} +import edu.uci.ics.texera.auth.{JwtAuthFilter, SessionUser} +import edu.uci.ics.texera.dao.SqlServer +import edu.uci.ics.texera.service.resource.{HealthCheckResource, AccessControlResource} +import io.dropwizard.auth.AuthDynamicFeature +import org.eclipse.jetty.server.session.SessionHandler +import org.jooq.impl.DSL + + +class AccessControlService extends Application[AccessControlServiceConfiguration] with LazyLogging { + override def initialize(bootstrap: Bootstrap[AccessControlServiceConfiguration]): Unit = { + // Register Scala module to Dropwizard default object mapper + bootstrap.getObjectMapper.registerModule(DefaultScalaModule) + + SqlServer.initConnection( + StorageConfig.jdbcUrl, + StorageConfig.jdbcUsername, + StorageConfig.jdbcPassword + ) + } + + override def run(configuration: AccessControlServiceConfiguration, environment: Environment): Unit = { + // Serve backend at /api + environment.jersey.setUrlPattern("/api/*") + + environment.jersey.register(classOf[SessionHandler]) + environment.servlets.setSessionHandler(new SessionHandler) + + environment.jersey.register(classOf[HealthCheckResource]) + environment.jersey.register(classOf[AccessControlResource]) + + // Register JWT authentication filter + environment.jersey.register(new AuthDynamicFeature(classOf[JwtAuthFilter])) + + // Enable @Auth annotation for injecting SessionUser + environment.jersey.register( + new io.dropwizard.auth.AuthValueFactoryProvider.Binder(classOf[SessionUser]) + ) + } +} +object AccessControlService { + def main(args: Array[String]): Unit = { + val accessControlPath = accessControlServicePath + .resolve("src") + .resolve("main") + .resolve("resources") + .resolve("access-control-service-web-config.yaml") + .toAbsolutePath + .toString + + // Start the Dropwizard application + new AccessControlService().run("server", accessControlPath) + } +} diff --git a/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/AccessControlServiceConfiguration.scala b/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/AccessControlServiceConfiguration.scala new file mode 100644 index 0000000000..1f388d8f9a --- /dev/null +++ b/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/AccessControlServiceConfiguration.scala @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package edu.uci.ics.texera.service + +import io.dropwizard.core.Configuration + +class AccessControlServiceConfiguration extends Configuration {} diff --git a/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/resource/AccessControlResource.scala b/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/resource/AccessControlResource.scala new file mode 100644 index 0000000000..8bca493850 --- /dev/null +++ b/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/resource/AccessControlResource.scala @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package edu.uci.ics.texera.service.resource + +import com.typesafe.scalalogging.LazyLogging +import edu.uci.ics.texera.auth.JwtParser.parseToken +import edu.uci.ics.texera.auth.SessionUser +import edu.uci.ics.texera.auth.util.{ComputingUnitAccess, HeaderField} +import edu.uci.ics.texera.dao.jooq.generated.enums.PrivilegeEnum +import jakarta.ws.rs.core._ +import jakarta.ws.rs.{GET, POST, Path, Produces} + +import java.util.Optional +import scala.jdk.CollectionConverters.{CollectionHasAsScala, MapHasAsScala} +import scala.util.matching.Regex + +object AccessControlResource extends LazyLogging { + + // Regex for the paths that require authorization + private val wsapiWorkflowWebsocket: Regex = """.*/wsapi/workflow-websocket.*""".r + private val apiExecutionsStats: Regex = """.*/api/executions/[0-9]+/stats/[0-9]+.*""".r + private val apiExecutionsResultExport: Regex = """.*/api/executions/result/export.*""".r + + /** + * Authorize the request based on the path and headers. + * @param uriInfo URI sent by Envoy or API Gateway + * @param headers HTTP headers sent by Envoy or API Gateway which include + * headers sent by the client (browser) + * @return HTTP Response with appropriate status code and headers + */ + def authorize(uriInfo: UriInfo, headers: HttpHeaders): Response = { + val path = uriInfo.getPath + logger.info(s"Authorizing request for path: $path") + + path match { + case wsapiWorkflowWebsocket() | apiExecutionsStats() | apiExecutionsResultExport() => + checkComputingUnitAccess(uriInfo, headers) + case _ => + logger.warn(s"No authorization logic for path: $path. Denying access.") + Response.status(Response.Status.FORBIDDEN).build() + } + } + + private def checkComputingUnitAccess(uriInfo: UriInfo, headers: HttpHeaders): Response = { + val queryParams: Map[String, String] = uriInfo + .getQueryParameters() + .asScala + .view + .mapValues(values => values.asScala.headOption.getOrElse("")) + .toMap + + logger.info(s"Request URI: ${uriInfo.getRequestUri} and headers: ${headers.getRequestHeaders.asScala} and queryParams: $queryParams") + + val token = queryParams.getOrElse( + "access-token", + headers + .getRequestHeader("Authorization") + .asScala + .headOption + .getOrElse("") + .replace("Bearer ", "") + ) + val cuid = queryParams.getOrElse("cuid", "") + val cuidInt = try { + cuid.toInt + } catch { + case _: NumberFormatException => + return Response.status(Response.Status.FORBIDDEN).build() + } + + var cuAccess: PrivilegeEnum = PrivilegeEnum.NONE + var userSession: Optional[SessionUser] = Optional.empty() + try { + userSession = parseToken(token) + if (userSession.isEmpty) + return Response.status(Response.Status.FORBIDDEN).build() + + val uid = userSession.get().getUid + cuAccess = ComputingUnitAccess.getComputingUnitAccess(cuidInt, uid) + if (cuAccess == PrivilegeEnum.NONE) + return Response.status(Response.Status.FORBIDDEN).build() + } catch { + case e: Exception => + return Response.status(Response.Status.FORBIDDEN).build() + } + + Response + .ok() + .header(HeaderField.UserComputingUnitAccess, cuAccess.toString) + .header(HeaderField.UserId, userSession.get().getUid.toString) + .header(HeaderField.UserName, userSession.get().getName) + .header(HeaderField.UserEmail, userSession.get().getEmail) + .build() + } +} +@Produces(Array(MediaType.APPLICATION_JSON)) +@Path("/auth") +class AccessControlResource extends LazyLogging { + + @GET + @Path("/{path:.*}") + def authorizeGet( + @Context uriInfo: UriInfo, + @Context headers: HttpHeaders + ): Response = { + AccessControlResource.authorize(uriInfo, headers) + } + + @POST + @Path("/{path:.*}") + def authorizePost( + @Context uriInfo: UriInfo, + @Context headers: HttpHeaders + ): Response = { + AccessControlResource.authorize(uriInfo, headers) + } +} diff --git a/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/resource/HealthCheckResource.scala b/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/resource/HealthCheckResource.scala new file mode 100644 index 0000000000..895f6a400a --- /dev/null +++ b/core/access-control-service/src/main/scala/edu/uci/ics/texera/service/resource/HealthCheckResource.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package edu.uci.ics.texera.service.resource + +import jakarta.ws.rs.core.MediaType +import jakarta.ws.rs.{GET, Path, Produces} + +@Path("/healthcheck") +@Produces(Array(MediaType.APPLICATION_JSON)) +class HealthCheckResource { + @GET + def healthCheck: Map[String, String] = Map("status" -> "ok") +} diff --git a/core/access-control-service/src/test/scala/edu/uci/ics/texera/AccessControlResourceSpec.scala b/core/access-control-service/src/test/scala/edu/uci/ics/texera/AccessControlResourceSpec.scala new file mode 100644 index 0000000000..349ec334aa --- /dev/null +++ b/core/access-control-service/src/test/scala/edu/uci/ics/texera/AccessControlResourceSpec.scala @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package edu.uci.ics.texera + +import edu.uci.ics.texera.auth.JwtAuth +import edu.uci.ics.texera.auth.util.HeaderField +import edu.uci.ics.texera.dao.MockTexeraDB +import edu.uci.ics.texera.dao.jooq.generated.enums.{PrivilegeEnum, UserRoleEnum, WorkflowComputingUnitTypeEnum} +import edu.uci.ics.texera.dao.jooq.generated.tables.daos.{ComputingUnitUserAccessDao, UserDao, WorkflowComputingUnitDao} +import edu.uci.ics.texera.dao.jooq.generated.tables.pojos.{ComputingUnitUserAccess, User, WorkflowComputingUnit} +import edu.uci.ics.texera.service.resource.AccessControlResource +import jakarta.ws.rs.core.{HttpHeaders, MultivaluedHashMap, Response, UriInfo} +import org.mockito.Mockito._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} + +import java.net.URI +import java.util + +class AccessControlResourceSpec extends AnyFlatSpec + with Matchers + with BeforeAndAfterAll + with BeforeAndAfterEach + with MockTexeraDB { + + private val testURI: String = "http://localhost:8080/"; + private val testPath: String = "/api/executions/1/stats/1" + + private val testUser1: User = { + val user = new User() + user.setUid(1) + user.setName("testuser") + user.setEmail("[email protected]") + user.setRole(UserRoleEnum.REGULAR) + user.setPassword("password") + user + } + + private val testUser2: User = { + val user = new User() + user.setUid(2) + user.setName("testuser2") + user.setEmail("[email protected]") + user.setRole(UserRoleEnum.REGULAR) + user.setPassword("password") + user + } + + private val testCU: WorkflowComputingUnit = { + val cu = new WorkflowComputingUnit() + cu.setUid(2) + cu.setType(WorkflowComputingUnitTypeEnum.kubernetes) + cu.setCuid(2) + cu.setName("test-cu") + cu + } + + private var token: String = _ + + override protected def beforeAll(): Unit = { + initializeDBAndReplaceDSLContext() + val userDao = new UserDao(getDSLContext.configuration()) + val computingUnitDao = new WorkflowComputingUnitDao(getDSLContext.configuration()) + val computingUnitOfUserDao = new ComputingUnitUserAccessDao(getDSLContext.configuration()) + + // insert user, computing unit, and access privilege into the mock database + userDao.insert(testUser1) + userDao.insert(testUser2) + computingUnitDao.insert(testCU) + + val cuAccess = new ComputingUnitUserAccess() + cuAccess.setUid(testUser1.getUid) + cuAccess.setCuid(testCU.getCuid) + cuAccess.setPrivilege(PrivilegeEnum.WRITE) + computingUnitOfUserDao.insert(cuAccess) + + val claims = JwtAuth.jwtClaims(testUser1, 1) + token = JwtAuth.jwtToken(claims) + } + + override protected def afterAll(): Unit = { + shutdownDB() + } + + "AccessControlResource" should "return FORBIDDEN for a GET request without a token" in { + val mockUriInfo = mock(classOf[UriInfo]) + val mockHttpHeaders = mock(classOf[HttpHeaders]) + val queryParams = new MultivaluedHashMap[String, String]() + queryParams.add("cuid", "1") + val requestHeaders = new MultivaluedHashMap[String, String]() + + when(mockUriInfo.getQueryParameters).thenReturn(queryParams) + when(mockUriInfo.getRequestUri).thenReturn(new URI(testURI)) + when(mockUriInfo.getPath).thenReturn(testPath) + when(mockHttpHeaders.getRequestHeaders).thenReturn(requestHeaders) + when(mockHttpHeaders.getRequestHeader("Authorization")).thenReturn(new util.ArrayList[String]()) + + val accessControlResource = new AccessControlResource() + val response = accessControlResource.authorizeGet(mockUriInfo, mockHttpHeaders) + + response.getStatus shouldBe Response.Status.FORBIDDEN.getStatusCode + } + + it should "return FORBIDDEN for a GET request with a non-integer cuid" in { + val mockUriInfo = mock(classOf[UriInfo]) + val mockHttpHeaders = mock(classOf[HttpHeaders]) + val queryParams = new MultivaluedHashMap[String, String]() + queryParams.add("cuid", "abc") + val requestHeaders = new MultivaluedHashMap[String, String]() + requestHeaders.add("Authorization", "Bearer dummy-token") + + when(mockUriInfo.getQueryParameters).thenReturn(queryParams) + when(mockUriInfo.getRequestUri).thenReturn(new URI(testURI)) + when(mockUriInfo.getPath).thenReturn(testPath) + when(mockHttpHeaders.getRequestHeaders).thenReturn(requestHeaders) + when(mockHttpHeaders.getRequestHeader("Authorization")).thenReturn(util.Arrays.asList("Bearer dummy-token")) + + val accessControlResource = new AccessControlResource() + val response = accessControlResource.authorizeGet(mockUriInfo, mockHttpHeaders) + + response.getStatus shouldBe Response.Status.FORBIDDEN.getStatusCode + } + + it should "return FORBIDDEN for a POST request without a token" in { + val mockUriInfo = mock(classOf[UriInfo]) + val mockHttpHeaders = mock(classOf[HttpHeaders]) + val queryParams = new MultivaluedHashMap[String, String]() + queryParams.add("cuid", "1") + val requestHeaders = new MultivaluedHashMap[String, String]() + + when(mockUriInfo.getQueryParameters).thenReturn(queryParams) + when(mockUriInfo.getRequestUri).thenReturn(new URI(testURI)) + when(mockUriInfo.getPath).thenReturn(testPath) + when(mockHttpHeaders.getRequestHeaders).thenReturn(requestHeaders) + when(mockHttpHeaders.getRequestHeader("Authorization")).thenReturn(new util.ArrayList[String]()) + + val accessControlResource = new AccessControlResource() + val response = accessControlResource.authorizePost(mockUriInfo, mockHttpHeaders) + + response.getStatus shouldBe Response.Status.FORBIDDEN.getStatusCode + } + + "AccessControlResource" should "return FORBIDDEN when user does not have access to the computing unit" in { + // Mock the request context + val mockUriInfo = mock(classOf[UriInfo]) + val mockHttpHeaders = mock(classOf[HttpHeaders]) + + // Prepare query parameters with a computing unit ID (cuid) + val queryParams = new MultivaluedHashMap[String, String]() + queryParams.add("cuid", "1") // Assuming user 1 does not have access to cuid 1 + + // Prepare request headers with the generated JWT + val requestHeaders = new MultivaluedHashMap[String, String]() + requestHeaders.add("Authorization", "Bearer " + token) + + // Stub the mock objects to return the prepared data + when(mockUriInfo.getQueryParameters).thenReturn(queryParams) + when(mockUriInfo.getRequestUri).thenReturn(new URI(testURI)) + when(mockUriInfo.getPath).thenReturn(testPath) + when(mockHttpHeaders.getRequestHeaders).thenReturn(requestHeaders) + when(mockHttpHeaders.getRequestHeader("Authorization")).thenReturn(util.Arrays.asList("Bearer " + token)) + + // Instantiate the resource and call the method under test + val accessControlResource = new AccessControlResource() + val response = accessControlResource.authorizeGet(mockUriInfo, mockHttpHeaders) + + // Assert that the response status is FORBIDDEN + response.getStatus shouldBe Response.Status.FORBIDDEN.getStatusCode + } + + it should "return OK and correct headers when user has access" in { + // Mock the request context + val mockUriInfo = mock(classOf[UriInfo]) + val mockHttpHeaders = mock(classOf[HttpHeaders]) + + // Prepare query parameters with a computing unit ID the user HAS access to + val queryParams = new MultivaluedHashMap[String, String]() + queryParams.add("cuid", testCU.getCuid.toString) + + // Prepare request headers with the generated JWT + val requestHeaders = new MultivaluedHashMap[String, String]() + requestHeaders.add("Authorization", "Bearer " + token) + + // Stub the mock objects to return the prepared data + when(mockUriInfo.getQueryParameters).thenReturn(queryParams) + when(mockUriInfo.getRequestUri).thenReturn(new URI(testURI)) + when(mockUriInfo.getPath).thenReturn(testPath) + when(mockHttpHeaders.getRequestHeaders).thenReturn(requestHeaders) + when(mockHttpHeaders.getRequestHeader("Authorization")).thenReturn(util.Arrays.asList("Bearer " + token)) + + // Instantiate the resource and call the method under test + val accessControlResource = new AccessControlResource() + val response = accessControlResource.authorizeGet(mockUriInfo, mockHttpHeaders) + + // Assert that the response status is OK and headers are correct + response.getStatus shouldBe Response.Status.OK.getStatusCode + response.getHeaderString( + HeaderField.UserComputingUnitAccess + ) shouldBe PrivilegeEnum.WRITE.toString + response.getHeaderString(HeaderField.UserId) shouldBe testUser1.getUid.toString + response.getHeaderString(HeaderField.UserName) shouldBe testUser1.getName + response.getHeaderString(HeaderField.UserEmail) shouldBe testUser1.getEmail + } +} \ No newline at end of file diff --git a/core/auth/src/main/scala/edu/uci/ics/texera/auth/util/ComputingUnitAccess.scala b/core/auth/src/main/scala/edu/uci/ics/texera/auth/util/ComputingUnitAccess.scala new file mode 100644 index 0000000000..c529edf8ec --- /dev/null +++ b/core/auth/src/main/scala/edu/uci/ics/texera/auth/util/ComputingUnitAccess.scala @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package edu.uci.ics.texera.auth.util + +import edu.uci.ics.texera.dao.SqlServer +import edu.uci.ics.texera.dao.jooq.generated.enums.PrivilegeEnum +import edu.uci.ics.texera.dao.jooq.generated.tables.daos.{ + ComputingUnitUserAccessDao, + WorkflowComputingUnitDao +} +import ComputingUnitAccess._ +import org.jooq.DSLContext + +import scala.jdk.CollectionConverters._ + +object ComputingUnitAccess { + private lazy val context: DSLContext = SqlServer + .getInstance() + .createDSLContext() + + def getComputingUnitAccess(cuid: Integer, uid: Integer): PrivilegeEnum = { + val workflowComputingUnitDao = new WorkflowComputingUnitDao(context.configuration()) + val unit = workflowComputingUnitDao.fetchOneByCuid(cuid) + + if (unit.getUid.equals(uid)) { + return PrivilegeEnum.WRITE // owner has write access + } + + val computingUnitUserAccessDao = new ComputingUnitUserAccessDao(context.configuration()) + val accessOpt = computingUnitUserAccessDao + .fetchByUid(uid) + .asScala + .find(_.getCuid.equals(cuid)) + + accessOpt match { + case Some(access) => access.getPrivilege + case None => PrivilegeEnum.NONE + } + } +} diff --git a/core/auth/src/main/scala/edu/uci/ics/texera/auth/util/HeaderField.scala b/core/auth/src/main/scala/edu/uci/ics/texera/auth/util/HeaderField.scala new file mode 100644 index 0000000000..2b98989737 --- /dev/null +++ b/core/auth/src/main/scala/edu/uci/ics/texera/auth/util/HeaderField.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package edu.uci.ics.texera.auth.util + +object HeaderField { + val UserComputingUnitAccess = "x-user-computing-unit-access" + val UserId = "x-user-id" + val UserName = "x-user-name" + val UserEmail = "x-user-email" +} diff --git a/core/build.sbt b/core/build.sbt index 658f755f33..dc4fa0e823 100644 --- a/core/build.sbt +++ b/core/build.sbt @@ -27,6 +27,16 @@ lazy val ConfigService = (project in file("config-service")) "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.17.0" ) ) +lazy val AccessControlService = (project in file("access-control-service")) + .dependsOn(Auth, Config, DAO) + .settings( + dependencyOverrides ++= Seq( + // override it as io.dropwizard 4 require 2.16.1 or higher + "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.17.0" + ) + ) + .configs(Test) + .dependsOn(DAO % "test->test", Auth % "test->test") lazy val WorkflowCore = (project in file("workflow-core")) .dependsOn(DAO, Config) .configs(Test) diff --git a/core/config/src/main/resources/default.conf b/core/config/src/main/resources/default.conf index 548a4882d3..5df68ecbfc 100644 --- a/core/config/src/main/resources/default.conf +++ b/core/config/src/main/resources/default.conf @@ -52,11 +52,10 @@ gui { } dataset { - # the file size limit for dataset upload single_file_upload_max_size_mib = 20 + max_number_of_concurrent_uploading_file = 3 - # the maximum number of file chunks that can be held in the memory; - # you may increase this number if your deployment environment has enough memory resource + # The maximum number of file chunks that can be held in the memory max_number_of_concurrent_uploading_file_chunks = 10 # the size of each chunk during the multipart upload of file diff --git a/core/config/src/main/scala/edu/uci/ics/amber/util/PathUtils.scala b/core/config/src/main/scala/edu/uci/ics/amber/util/PathUtils.scala index a4d82a9030..827bd5e289 100644 --- a/core/config/src/main/scala/edu/uci/ics/amber/util/PathUtils.scala +++ b/core/config/src/main/scala/edu/uci/ics/amber/util/PathUtils.scala @@ -63,6 +63,8 @@ object PathUtils { lazy val configServicePath: Path = corePath.resolve("config-service") + lazy val accessControlServicePath: Path = corePath.resolve("access-control-service") + private lazy val datasetsRootPath = corePath.resolve("amber").resolve("user-resources").resolve("datasets") diff --git a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html index 8438ab7213..21f42c8a09 100644 --- a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html +++ b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html @@ -272,6 +272,18 @@ </div> </ng-template> + <div class="settings-row"> + <span>Concurrent Files:</span> + <nz-input-number + [(ngModel)]="maxConcurrentFiles" + [nzMin]="1" + [nzMax]="1000" + [nzStep]="1" + [nzPrecision]="0"> + </nz-input-number> + </div> + <div class="help-text-number">Number of files that can be uploaded simultaneously. (Range: 1 - 1000)</div> + <div class="settings-row"> <span>File Size:</span> <nz-input-number diff --git a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts index f7d5186ec3..52a8c5d2e0 100644 --- a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts +++ b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts @@ -48,6 +48,7 @@ export class AdminSettingsComponent implements OnInit { about_enabled: false, }; + maxConcurrentFiles: number = 3; maxFileSizeMiB: number = 20; maxConcurrentChunks: number = 10; chunkSizeMiB: number = 50; @@ -97,6 +98,10 @@ export class AdminSettingsComponent implements OnInit { } private loadDatasetSettings(): void { + this.adminSettingsService + .getSetting("max_number_of_concurrent_uploading_file") + .pipe(untilDestroyed(this)) + .subscribe(value => (this.maxConcurrentFiles = parseInt(value))); this.adminSettingsService .getSetting("single_file_upload_max_size_mib") .pipe(untilDestroyed(this)) @@ -203,7 +208,12 @@ export class AdminSettingsComponent implements OnInit { } saveDatasetSettings(): void { - if (this.maxFileSizeMiB < 1 || this.maxConcurrentChunks < 1 || this.chunkSizeMiB < 1) { + if ( + this.maxFileSizeMiB < 1 || + this.maxConcurrentFiles < 1 || + this.maxConcurrentChunks < 1 || + this.chunkSizeMiB < 1 + ) { this.message.error("Please enter valid integer values."); return; } @@ -217,6 +227,10 @@ export class AdminSettingsComponent implements OnInit { } const saveRequests = [ + this.adminSettingsService.updateSetting( + "max_number_of_concurrent_uploading_file", + this.maxConcurrentFiles.toString() + ), this.adminSettingsService.updateSetting("single_file_upload_max_size_mib", this.maxFileSizeMiB.toString()), this.adminSettingsService.updateSetting( "max_number_of_concurrent_uploading_file_chunks", @@ -235,6 +249,7 @@ export class AdminSettingsComponent implements OnInit { resetDatasetSettings(): void { [ + "max_number_of_concurrent_uploading_file", "single_file_upload_max_size_mib", "max_number_of_concurrent_uploading_file_chunks", "multipart_upload_chunk_size_mib", diff --git a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.html b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.html index 622124625c..d2b01ac115 100644 --- a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.html +++ b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.html @@ -276,54 +276,90 @@ nzActive="true" nzHeader="Create New Version"> <texera-user-files-uploader (uploadedFiles)="onNewUploadFilesChanged($event)"> </texera-user-files-uploader> - <div class="upload-progress-wrapper"> - <div - *ngFor="let task of uploadTasks; trackBy: trackByTask" - class="upload-progress-container"> - <div class="progress-header"> - <span><b>{{ task.status }}</b>: {{ task.filePath }}</span> - <button - nz-button - nzType="text" - nzShape="circle" - [nz-tooltip]=" + + <nz-collapse + nzGhost + class="upload-status-panels"> + <nz-collapse-panel + [nzHeader]="queuedCount > 0 + ? ('Pending: ' + queuedCount + ' file(s)') + : 'Pending'"> + <div + *ngIf="queuedCount > 0" + class="upload-progress-wrapper-pending"> + <div + *ngFor="let fileName of queuedFileNames" + class="upload-progress-container"> + <span>{{ fileName }}</span> + </div> + </div> + </nz-collapse-panel> + <nz-divider class="section-divider"></nz-divider> + + <nz-collapse-panel + [nzHeader]="activeCount > 0 + ? ('Uploading: ' + activeCount + ' file(s)') + : 'Uploading'"> + <div + *ngIf="activeCount > 0" + class="upload-progress-wrapper"> + <div + *ngFor="let task of uploadTasks; trackBy: trackByTask" + class="upload-progress-container"> + <div class="progress-header"> + <span><b>{{ task.status }}</b>: {{ task.filePath }}</span> + <button + nz-button + nzType="text" + nzShape="circle" + [nz-tooltip]=" (task.status === 'aborted' || task.status === 'finished') ? 'Close' : 'Cancel the upload' " - (click)="onClickAbortUploadProgress(task)"> - <i - nz-icon - nzType="close" - nzTheme="outline"></i> - </button> - </div> - <div - class="upload-stats" - *ngIf="task.status !== 'initializing'"> - <nz-progress - [nzPercent]="task.percentage" - [nzStatus]="getUploadStatus(task.status)"></nz-progress> - <nz-tag - *ngIf="task.status === 'uploading'" - [nzColor]="'blue'"> - <span class="fixed-width-speed">{{ formatSpeed(task.uploadSpeed) }}</span> - - <span class="fixed-width-time">{{ formatTime(task.totalTime ?? 0) }}</span> elapsed, - <span class="fixed-width-time">{{ formatTime(task.estimatedTimeRemaining ?? 0) }} left</span> - </nz-tag> + (click)="onClickAbortUploadProgress(task)"> + <i + nz-icon + nzType="close" + nzTheme="outline"></i> + </button> + </div> + + <div + class="upload-stats" + *ngIf="task.status !== 'initializing'"> + <nz-progress + [nzPercent]="task.percentage" + [nzStatus]="getUploadStatus(task.status)"></nz-progress> + <nz-tag + *ngIf="task.status === 'uploading'" + [nzColor]="'blue'"> + <span class="fixed-width-speed">{{ formatSpeed(task.uploadSpeed) }}</span> - + <span class="fixed-width-time">{{ formatTime(task.totalTime ?? 0) }}</span> elapsed, + <span class="fixed-width-time">{{ formatTime(task.estimatedTimeRemaining ?? 0) }} left</span> + </nz-tag> - <nz-tag *ngIf="(task.status === 'finished' || task.status === 'aborted')"> - Upload time: {{ formatTime(task.totalTime ?? 0) }} - </nz-tag> + <nz-tag *ngIf="(task.status === 'finished' || task.status === 'aborted')"> + Upload time: {{ formatTime(task.totalTime ?? 0) }} + </nz-tag> + </div> + </div> </div> - </div> - </div> + </nz-collapse-panel> + <nz-divider class="section-divider"></nz-divider> - <texera-dataset-staged-objects-list - [uploadTimeMap]="uploadTimeMap" - [did]="did" - [userMakeChangesEvent]="userMakeChanges" - (stagedObjectsChanged)="onStagedObjectsUpdated($event)"></texera-dataset-staged-objects-list> + <nz-collapse-panel + [nzHeader]="pendingChangesCount > 0 + ? ('Finished: ' + pendingChangesCount + ' file(s)') + : 'Finished'" + [nzActive]="false"> + <texera-dataset-staged-objects-list + [uploadTimeMap]="uploadTimeMap" + [did]="did" + [userMakeChangesEvent]="userMakeChanges" + (stagedObjectsChanged)="onStagedObjectsUpdated($event)"></texera-dataset-staged-objects-list> + </nz-collapse-panel> + </nz-collapse> <div *ngIf="userHasWriteAccess() && userHasPendingChanges" class="version-creator"> diff --git a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss index 6e40560aa0..0790f28358 100644 --- a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss +++ b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss @@ -170,7 +170,9 @@ nz-select { margin-top: 15%; } -.upload-progress-wrapper { +.upload-progress-wrapper, +.upload-progress-wrapper-pending { + margin-top: 5px; max-height: 25vh; overflow-y: auto; padding-right: 4px; @@ -180,6 +182,13 @@ nz-select { margin-left: 20px; } +.upload-progress-wrapper-pending { + display: flex; + flex-direction: column; + gap: 10px; + max-height: 15vh; +} + .version-creator { margin-top: 20px; padding: 40px; @@ -233,6 +242,10 @@ nz-select { .upload-stats { font-size: 13px; margin-bottom: 20px; + nz-progress { + width: 97%; + display: inline-block; + } } :host ::ng-deep .upload-stats .ant-tag { @@ -250,3 +263,7 @@ nz-select { min-width: 2ch; text-align: right; } + +.section-divider { + margin: 8px 0; +} diff --git a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts index 7b7f7947f3..a7ae1e17d8 100644 --- a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts +++ b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts @@ -85,12 +85,19 @@ export class DatasetDetailComponent implements OnInit { public displayPreciseViewCount = false; userHasPendingChanges: boolean = false; + pendingChangesCount: number = 0; + // Uploading setting chunkSizeMiB: number = 50; maxConcurrentChunks: number = 10; private uploadSubscriptions = new Map<string, Subscription>(); uploadTimeMap = new Map<string, number>(); + // Cap number of concurrent files uploads + maxConcurrentFiles: number = 3; + private activeUploads: number = 0; + private pendingQueue: Array<{ fileName: string; startUpload: () => void }> = []; + versionName: string = ""; isCreatingVersion: boolean = false; @@ -100,7 +107,6 @@ export class DatasetDetailComponent implements OnInit { filePath: string; } > = []; - private autoHideTimers: number[] = []; @Output() userMakeChanges = new EventEmitter<void>(); @@ -329,6 +335,7 @@ export class DatasetDetailComponent implements OnInit { onStagedObjectsUpdated(stagedObjects: DatasetStagedObject[]) { this.userHasPendingChanges = stagedObjects.length > 0; + this.pendingChangesCount = stagedObjects.length; } onVersionSelected(version: DatasetVersion): void { @@ -385,95 +392,157 @@ export class DatasetDetailComponent implements OnInit { .getSetting("max_number_of_concurrent_uploading_file_chunks") .pipe(untilDestroyed(this)) .subscribe(value => (this.maxConcurrentChunks = parseInt(value))); + this.adminSettingsService + .getSetting("max_number_of_concurrent_uploading_file") + .pipe(untilDestroyed(this)) + .subscribe(value => { + this.maxConcurrentFiles = parseInt(value); + }); } onNewUploadFilesChanged(files: FileUploadItem[]) { if (this.did) { - files.forEach((file, idx) => { - // Cancel any existing upload for the same file to prevent progress confusion - this.uploadSubscriptions.get(file.name)?.unsubscribe(); - this.uploadSubscriptions.delete(file.name); - this.uploadTasks = this.uploadTasks.filter(t => t.filePath !== file.name); - - // Add an initializing task placeholder to uploadTasks - this.uploadTasks.push({ - filePath: file.name, - percentage: 0, - status: "initializing", - uploadId: "", - physicalAddress: "", - }); - // Start multipart upload - const subscription = this.datasetService - .multipartUpload( - this.datasetName, - file.name, - file.file, - this.chunkSizeMiB * 1024 * 1024, - this.maxConcurrentChunks - ) - .pipe(untilDestroyed(this)) - .subscribe({ - next: progress => { - // Find the task - const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); - - if (taskIndex !== -1) { - // Update the task with new progress info - this.uploadTasks[taskIndex] = { - ...this.uploadTasks[taskIndex], - ...progress, - percentage: progress.percentage ?? this.uploadTasks[taskIndex].percentage ?? 0, - }; - - // Auto‑hide when upload is truly finished - if (progress.status === "finished" && progress.totalTime) { - const filename = file.name.split("/").pop() || file.name; - this.uploadTimeMap.set(filename, progress.totalTime); + files.forEach(file => { + // Check if currently uploading + this.cancelExistingUpload(file.name); + + // Create upload function + const startUpload = () => { + this.pendingQueue = this.pendingQueue.filter(item => item.fileName !== file.name); + + // Add an initializing task placeholder to uploadTasks + this.uploadTasks.unshift({ + filePath: file.name, + percentage: 0, + status: "initializing", + uploadId: "", + physicalAddress: "", + }); + // Start multipart upload + const subscription = this.datasetService + .multipartUpload( + this.datasetName, + file.name, + file.file, + this.chunkSizeMiB * 1024 * 1024, + this.maxConcurrentChunks + ) + .pipe(untilDestroyed(this)) + .subscribe({ + next: progress => { + // Find the task + const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); + + if (taskIndex !== -1) { + // Update the task with new progress info + this.uploadTasks[taskIndex] = { + ...this.uploadTasks[taskIndex], + ...progress, + percentage: progress.percentage ?? this.uploadTasks[taskIndex].percentage ?? 0, + }; + + // Auto-hide when upload is truly finished + if (progress.status === "finished" && progress.totalTime) { + const filename = file.name.split("/").pop() || file.name; + this.uploadTimeMap.set(filename, progress.totalTime); + this.userMakeChanges.emit(); + this.scheduleHide(taskIndex); + this.onUploadComplete(); + } + } + }, + error: () => { + // Handle upload error + const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); + + if (taskIndex !== -1) { + this.uploadTasks[taskIndex] = { + ...this.uploadTasks[taskIndex], + percentage: 100, + status: "aborted", + }; + this.scheduleHide(taskIndex); + } + this.onUploadComplete(); + }, + complete: () => { + const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); + if (taskIndex !== -1 && this.uploadTasks[taskIndex].status !== "finished") { + this.uploadTasks[taskIndex].status = "finished"; this.userMakeChanges.emit(); this.scheduleHide(taskIndex); + this.onUploadComplete(); } - } - }, - error: () => { - // Handle upload error - const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); - - if (taskIndex !== -1) { - this.uploadTasks[taskIndex] = { - ...this.uploadTasks[taskIndex], - percentage: 100, - status: "aborted", - }; - this.scheduleHide(taskIndex); - } - }, - complete: () => { - const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); - if (taskIndex !== -1 && this.uploadTasks[taskIndex].status !== "finished") { - this.uploadTasks[taskIndex].status = "finished"; - this.userMakeChanges.emit(); - this.scheduleHide(taskIndex); - } - }, - }); - // Store the subscription for later cleanup - this.uploadSubscriptions.set(file.name, subscription); + }, + }); + // Store the subscription for later cleanup + this.uploadSubscriptions.set(file.name, subscription); + }; + + // Queue management + if (this.activeUploads < this.maxConcurrentFiles) { + this.activeUploads++; + startUpload(); + } else { + this.pendingQueue.push({ fileName: file.name, startUpload }); + } }); } } - // Hide a task row after 5s (stores timer to clear on destroy) and clean up its subscription + private cancelExistingUpload(fileName: string): void { + const isUploading = this.uploadTasks.some( + t => t.filePath === fileName && (t.status === "uploading" || t.status === "initializing") + ); + this.uploadSubscriptions.get(fileName)?.unsubscribe(); + this.uploadSubscriptions.delete(fileName); + this.uploadTasks = this.uploadTasks.filter(t => t.filePath !== fileName); + + // Process next in queue if this was active + if (isUploading) { + this.onUploadComplete(); + } + // Remove from pending queue if present + this.pendingQueue = this.pendingQueue.filter(item => item.fileName !== fileName); + } + + private processNextQueuedUpload(): void { + if (this.pendingQueue.length > 0 && this.activeUploads < this.maxConcurrentFiles) { + const next = this.pendingQueue.shift(); + if (next) { + this.activeUploads++; + next.startUpload(); + } + } + } + + private onUploadComplete(): void { + this.activeUploads--; + this.processNextQueuedUpload(); + } + + get queuedFileNames(): string[] { + return this.pendingQueue.map(item => item.fileName); + } + + get queuedCount(): number { + return this.pendingQueue.length; + } + + get activeCount(): number { + return this.activeUploads; + } + + // Hide a task row after 5s private scheduleHide(idx: number) { if (idx === -1) { return; } const key = this.uploadTasks[idx].filePath; this.uploadSubscriptions.delete(key); - const handle = window.setTimeout(() => { + setTimeout(() => { this.uploadTasks = this.uploadTasks.filter(t => t.filePath !== key); }, 5000); - this.autoHideTimers.push(handle); } onClickAbortUploadProgress(task: MultipartUploadProgress & { filePath: string }) { @@ -482,6 +551,11 @@ export class DatasetDetailComponent implements OnInit { subscription.unsubscribe(); this.uploadSubscriptions.delete(task.filePath); } + + if (task.status === "uploading" || task.status === "initializing") { + this.onUploadComplete(); + } + this.datasetService .finalizeMultipartUpload( this.datasetName, diff --git a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.html b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.html index 5b1dece350..a1820dcecf 100644 --- a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.html +++ b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.html @@ -19,7 +19,6 @@ <div class="staged-object-list-container"> <nz-list - nzBordered nzSize="small" *ngIf="datasetStagedObjects.length > 0"> <nz-list-item *ngFor="let obj of datasetStagedObjects"> diff --git a/core/gui/src/app/dashboard/component/user/user-quota/user-quota.component.html b/core/gui/src/app/dashboard/component/user/user-quota/user-quota.component.html index 5df5793baf..51a1dcbf69 100644 --- a/core/gui/src/app/dashboard/component/user/user-quota/user-quota.component.html +++ b/core/gui/src/app/dashboard/component/user/user-quota/user-quota.component.html @@ -66,20 +66,6 @@ <p class="info-content">{{ formatSize(this.totalQuotaSize) }}</p> </div> </div> - <nz-card - class="section-title" - [style.backgroundColor]="backgroundColor"> - <h2 - class="page-title" - [style.color]="textColor"> - Diagram - </h2> - </nz-card> - <div class="charts-grid"> - <div id="sizePieChart"></div> - <div id="datasetLineChart"></div> - <div id="workflowLineChart"></div> - </div> </nz-tab> <nz-tab nzTitle="Result Cache"> <nz-collapse> @@ -124,6 +110,15 @@ </nz-collapse-panel> </nz-collapse> </nz-tab> + <nz-tab + nzTitle="Diagrams" + nzForceRender="true"> + <div class="charts-grid"> + <div id="sizePieChart"></div> + <div id="datasetLineChart"></div> + <div id="workflowLineChart"></div> + </div> + </nz-tab> </nz-tabset> </div> </div> diff --git a/core/gui/src/app/dashboard/component/user/user-quota/user-quota.component.scss b/core/gui/src/app/dashboard/component/user/user-quota/user-quota.component.scss index d5b32caba8..6efa34ba61 100644 --- a/core/gui/src/app/dashboard/component/user/user-quota/user-quota.component.scss +++ b/core/gui/src/app/dashboard/component/user/user-quota/user-quota.component.scss @@ -49,7 +49,8 @@ display: flex; flex-wrap: wrap; justify-content: center; - padding-top: 40px; + height: 70vh; + overflow-y: auto; > div { display: flex; @@ -59,7 +60,6 @@ background: #fff; box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); padding: 10px; - height: 100%; &:hover { box-shadow: 0 0 15px rgba(0, 0, 0, 0.2); diff --git a/core/gui/src/app/dashboard/service/user/download/download.service.ts b/core/gui/src/app/dashboard/service/user/download/download.service.ts index 9c19cf19e2..f02a2411b5 100644 --- a/core/gui/src/app/dashboard/service/user/download/download.service.ts +++ b/core/gui/src/app/dashboard/service/user/download/download.service.ts @@ -58,8 +58,8 @@ export class DownloadService { downloadWorkflow(id: number, name: string): Observable<DownloadableItem> { return this.workflowPersistService.retrieveWorkflow(id).pipe( - map(({ wid, creationTime, lastModifiedTime, ...workflowCopy }) => { - const workflowJson = JSON.stringify({ ...workflowCopy, readonly: false }); + map(({ content }) => { + const workflowJson = JSON.stringify(content, null, 2); const fileName = `${name}.json`; const blob = new Blob([workflowJson], { type: "text/plain;charset=utf-8" }); return { blob, fileName }; diff --git a/core/gui/src/app/workspace/component/left-panel/settings/settings.component.ts b/core/gui/src/app/workspace/component/left-panel/settings/settings.component.ts index 39547bda62..ead28857ff 100644 --- a/core/gui/src/app/workspace/component/left-panel/settings/settings.component.ts +++ b/core/gui/src/app/workspace/component/left-panel/settings/settings.component.ts @@ -55,6 +55,12 @@ export class SettingsComponent implements OnInit { dataTransferBatchSize: [this.currentDataTransferBatchSize, [Validators.required, Validators.min(1)]], }); + this.settingsForm.valueChanges.pipe(untilDestroyed(this)).subscribe(value => { + if (this.settingsForm.valid) { + this.confirmUpdateDataTransferBatchSize(value.dataTransferBatchSize); + } + }); + this.workflowActionService .workflowChanged() .pipe(untilDestroyed(this)) diff --git a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts index cde1da2742..cc42b86325 100644 --- a/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/core/gui/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -188,7 +188,7 @@ export class ComputingUnitSelectionComponent implements OnInit { /** * Registers a subscription to listen for workflow metadata changes; - * Calls `onComputingUnitChange` when the `wid` changes; + * Calls `selectComputingUnit` when the `wid` changes; * The wid can change by time because of the workspace rendering; */ private registerWorkflowMetadataSubscription(): void { diff --git a/core/gui/src/app/workspace/component/workflow-editor/workflow-editor.component.ts b/core/gui/src/app/workspace/component/workflow-editor/workflow-editor.component.ts index 089ea9eead..e46bb9acec 100644 --- a/core/gui/src/app/workspace/component/workflow-editor/workflow-editor.component.ts +++ b/core/gui/src/app/workspace/component/workflow-editor/workflow-editor.component.ts @@ -17,14 +17,19 @@ * under the License. */ -import { AfterViewInit, ChangeDetectorRef, Component, ElementRef, OnDestroy } from "@angular/core"; +import { OnInit, AfterViewInit, ChangeDetectorRef, Component, ElementRef, OnDestroy } from "@angular/core"; import { fromEvent, merge, Subject } from "rxjs"; import { NzModalCommentBoxComponent } from "./comment-box-modal/nz-modal-comment-box.component"; import { NzModalRef, NzModalService } from "ng-zorro-antd/modal"; import { DragDropService } from "../../service/drag-drop/drag-drop.service"; import { DynamicSchemaService } from "../../service/dynamic-schema/dynamic-schema.service"; import { ExecuteWorkflowService } from "../../service/execute-workflow/execute-workflow.service"; -import { fromJointPaperEvent, JointUIService, linkPathStrokeColor } from "../../service/joint-ui/joint-ui.service"; +import { + deleteButtonPath, + fromJointPaperEvent, + JointUIService, + linkPathStrokeColor, +} from "../../service/joint-ui/joint-ui.service"; import { ValidationWorkflowService } from "../../service/validation/validation-workflow.service"; import { WorkflowActionService } from "../../service/workflow-graph/model/workflow-action.service"; import { WorkflowStatusService } from "../../service/workflow-status/workflow-status.service"; @@ -82,7 +87,7 @@ export const MAIN_CANVAS = { templateUrl: "workflow-editor.component.html", styleUrls: ["workflow-editor.component.scss"], }) -export class WorkflowEditorComponent implements AfterViewInit, OnDestroy { +export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy { editor!: HTMLElement; editorWrapper!: HTMLElement; paper!: joint.dia.Paper; @@ -91,6 +96,8 @@ export class WorkflowEditorComponent implements AfterViewInit, OnDestroy { private _onProcessKeyboardActionObservable: Subject<void> = new Subject(); private wrapper; private currentOpenedOperatorID: string | null = null; + private removeButton!: new () => joint.linkTools.Button; + private breakpointButton!: new () => joint.linkTools.Button; constructor( private workflowActionService: WorkflowActionService, @@ -114,6 +121,12 @@ export class WorkflowEditorComponent implements AfterViewInit, OnDestroy { this.wrapper = this.workflowActionService.getJointGraphWrapper(); } + ngOnInit(): void { + // Cache the tool constructors + this.removeButton = WorkflowEditorComponent.getRemoveButton(); + this.breakpointButton = WorkflowEditorComponent.getBreakpointButton(); + } + /** * This function is provided to JointJS to disallow links starting from an in port. * @@ -1086,18 +1099,17 @@ export class WorkflowEditorComponent implements AfterViewInit, OnDestroy { fromJointPaperEvent(this.paper, "link:mouseenter") .pipe(map(value => value[0])) .pipe(untilDestroyed(this)) - .subscribe(elementView => { + .subscribe(linkView => { + // Create an array to hold the tools + const tools: joint.dia.ToolView[] = [new this.removeButton()]; + + // If breakpoints are enabled, also add the breakpoint button if (this.config.env.linkBreakpointEnabled) { - this.paper.getModelById(elementView.model.id).attr({ - ".tool-remove": { display: "block" }, - }); - this.paper.getModelById(elementView.model.id).findView(this.paper).showTools(); - } else { - // only display the delete button - this.paper.getModelById(elementView.model.id).attr({ - ".tool-remove": { display: "block" }, - }); + tools.push(new this.breakpointButton()); } + + const toolsView = new joint.dia.ToolsView({ tools }); + linkView.addTools(toolsView); }); /** @@ -1139,7 +1151,7 @@ export class WorkflowEditorComponent implements AfterViewInit, OnDestroy { .pipe(this.wrapper.jointGraphContext.bufferWhileAsync, untilDestroyed(this)) .subscribe(link => { const linkView = link.findView(this.paper); - const breakpointButtonTool = this.jointUIService.getBreakpointButton(); + const breakpointButtonTool = this.breakpointButton; const breakpointButton = new breakpointButtonTool(); const toolsView = new joint.dia.ToolsView({ name: "basic-tools", @@ -1362,4 +1374,98 @@ export class WorkflowEditorComponent implements AfterViewInit, OnDestroy { this.paper.translate(-targetCoord.x, -targetCoord.y); }); } + + /** + * Info button on link between operator shown when user hovers over links + */ + private static getBreakpointButton(): new () => joint.linkTools.Button { + return joint.linkTools.Button.extend({ + name: "info-button", + options: { + markup: [ + { + tagName: "circle", + selector: "info-button", + attributes: { + r: 10, + fill: "#001DFF", + cursor: "pointer", + }, + }, + { + tagName: "path", + selector: "icon", + attributes: { + d: "M -2 4 2 4 M 0 3 0 0 M -2 -1 1 -1 M -1 -4 1 -4", + fill: "none", + stroke: "#FFFFFF", + "stroke-width": 2, + "pointer-events": "none", + }, + }, + ], + distance: -60, + offset: 0, + action: function (event: JQuery.Event, linkView: joint.dia.LinkView) { + // when this button is clicked, it triggers an joint paper event + if (linkView.paper) { + linkView.paper.trigger("tool:breakpoint", linkView, event); + } + }, + }, + }); + } + + /** + * Remove button on link between operator shown when user hovers over links + */ + private static RemoveButton: new () => joint.linkTools.Button; + + private static getRemoveButton(): new () => joint.linkTools.Button { + // Check if the class has already been created. + if (!WorkflowEditorComponent.RemoveButton) { + // If not, create it once and store it in the static property. + WorkflowEditorComponent.RemoveButton = joint.linkTools.Button.extend({ + name: "remove-button", + options: { + markup: [ + { + tagName: "circle", + selector: "button", + attributes: { + r: 10, + fill: "none", + stroke: "#D8656A", + "stroke-width": 2, + "pointer-events": "visibleStroke", + cursor: "pointer", + }, + }, + { + tagName: "path", + selector: "icon", + attributes: { + d: "M -4 -4 L 4 4 M 4 -4 L -4 4", + fill: "none", + stroke: "#D8656A", + "stroke-width": 2, + "stroke-linecap": "round", + "pointer-events": "none", + }, + }, + ], + distance: -90, + offset: 0, + action: function (evt: JQuery.Event, linkView: joint.dia.LinkView) { + if (linkView.paper) { + linkView.paper.trigger("tool:remove", linkView, evt); + } + }, + }, + }); + } + + // Return the cached class. + return WorkflowEditorComponent.RemoveButton; + } } diff --git a/core/gui/src/app/workspace/service/computing-unit-status/computing-unit-status.service.ts b/core/gui/src/app/workspace/service/computing-unit-status/computing-unit-status.service.ts index b5c840c866..4604e52c6f 100644 --- a/core/gui/src/app/workspace/service/computing-unit-status/computing-unit-status.service.ts +++ b/core/gui/src/app/workspace/service/computing-unit-status/computing-unit-status.service.ts @@ -51,6 +51,7 @@ export class ComputingUnitStatusService implements OnDestroy { private readonly REFRESH_INTERVAL_MS = 2000; private refreshSubscription: Subscription | null = null; private currentConnectedCuid?: number; + private currentConnectedWid?: number; private selectedUnitPoll?: Subscription; constructor( @@ -148,19 +149,23 @@ export class ComputingUnitStatusService implements OnDestroy { }); } + // /** * Select a computing unit **by its CUID** and emit the updated selection. */ public selectComputingUnit(wid: number | undefined, cuid: number): void { const trySelect = (unit: DashboardWorkflowComputingUnit) => { // open websocket if needed - if (isDefined(wid) && this.currentConnectedCuid !== cuid) { + const shouldReconnect = this.currentConnectedCuid !== cuid || this.currentConnectedWid !== wid; + if (isDefined(wid) && shouldReconnect) { if (this.workflowWebsocketService.isConnected) { this.workflowWebsocketService.closeWebsocket(); this.workflowStatusService.clearStatus(); } + this.workflowWebsocketService.openWebsocket(wid, this.userService.getCurrentUser()?.uid, cuid); this.currentConnectedCuid = cuid; + this.currentConnectedWid = wid; this.selectedUnitSubject.next(unit); this.startPollingSelectedUnit(cuid); } diff --git a/core/gui/src/app/workspace/service/joint-ui/joint-ui.service.ts b/core/gui/src/app/workspace/service/joint-ui/joint-ui.service.ts index 700594ab2d..ff0c930ab7 100644 --- a/core/gui/src/app/workspace/service/joint-ui/joint-ui.service.ts +++ b/core/gui/src/app/workspace/service/joint-ui/joint-ui.service.ts @@ -472,44 +472,6 @@ export class JointUIService { jointPaper.getModelById(operator.operatorID).attr(`.${operatorNameClass}/text`, displayName); } - public getBreakpointButton(): new () => joint.linkTools.Button { - return joint.linkTools.Button.extend({ - name: "info-button", - options: { - markup: [ - { - tagName: "circle", - selector: "info-button", - attributes: { - r: 10, - fill: "#001DFF", - cursor: "pointer", - }, - }, - { - tagName: "path", - selector: "icon", - attributes: { - d: "M -2 4 2 4 M 0 3 0 0 M -2 -1 1 -1 M -1 -4 1 -4", - fill: "none", - stroke: "#FFFFFF", - "stroke-width": 2, - "pointer-events": "none", - }, - }, - ], - distance: 60, - offset: 0, - action: function (event: JQuery.Event, linkView: joint.dia.LinkView) { - // when this button is clicked, it triggers an joint paper event - if (linkView.paper) { - linkView.paper.trigger("tool:breakpoint", linkView, event); - } - }, - }, - }); - } - public getCommentElement(commentBox: CommentBox): joint.dia.Element { const basic = new joint.shapes.standard.Rectangle(); if (commentBox.commentBoxPosition) basic.position(commentBox.commentBoxPosition.x, commentBox.commentBoxPosition.y); diff --git a/core/gui/src/assets/operator_images/SklearnTrainingAdaptiveBoosting.png b/core/gui/src/assets/operator_images/SklearnTrainingAdaptiveBoosting.png new file mode 100644 index 0000000000..2daaf54222 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingAdaptiveBoosting.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingBagging.png b/core/gui/src/assets/operator_images/SklearnTrainingBagging.png new file mode 100644 index 0000000000..debd6fec78 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingBagging.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingBernoulliNaiveBayes.png b/core/gui/src/assets/operator_images/SklearnTrainingBernoulliNaiveBayes.png new file mode 100644 index 0000000000..736f562180 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingBernoulliNaiveBayes.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingComplementNaiveBayes.png b/core/gui/src/assets/operator_images/SklearnTrainingComplementNaiveBayes.png new file mode 100644 index 0000000000..8c048f0b00 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingComplementNaiveBayes.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingDecisionTree.png b/core/gui/src/assets/operator_images/SklearnTrainingDecisionTree.png new file mode 100644 index 0000000000..60d0b815c3 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingDecisionTree.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingDummy.png b/core/gui/src/assets/operator_images/SklearnTrainingDummy.png new file mode 100644 index 0000000000..203f4a1e68 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingDummy.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingExtraTree.png b/core/gui/src/assets/operator_images/SklearnTrainingExtraTree.png new file mode 100644 index 0000000000..273c719535 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingExtraTree.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingExtraTrees.png b/core/gui/src/assets/operator_images/SklearnTrainingExtraTrees.png new file mode 100644 index 0000000000..42c77aa5e5 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingExtraTrees.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingGaussianNaiveBayes.png b/core/gui/src/assets/operator_images/SklearnTrainingGaussianNaiveBayes.png new file mode 100644 index 0000000000..09c1f13acc Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingGaussianNaiveBayes.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingGradientBoosting.png b/core/gui/src/assets/operator_images/SklearnTrainingGradientBoosting.png new file mode 100644 index 0000000000..980f5910c8 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingGradientBoosting.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingKNN.png b/core/gui/src/assets/operator_images/SklearnTrainingKNN.png new file mode 100644 index 0000000000..23e0477686 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingKNN.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingLinearRegression.png b/core/gui/src/assets/operator_images/SklearnTrainingLinearRegression.png new file mode 100644 index 0000000000..5ee01e7e47 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingLinearRegression.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingLinearSVM.png b/core/gui/src/assets/operator_images/SklearnTrainingLinearSVM.png new file mode 100644 index 0000000000..510d391429 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingLinearSVM.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingLogisticRegression.png b/core/gui/src/assets/operator_images/SklearnTrainingLogisticRegression.png new file mode 100644 index 0000000000..4c598becc9 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingLogisticRegression.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingLogisticRegressionCV.png b/core/gui/src/assets/operator_images/SklearnTrainingLogisticRegressionCV.png new file mode 100644 index 0000000000..d7dbc742e3 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingLogisticRegressionCV.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingMultiLayerPerceptron.png b/core/gui/src/assets/operator_images/SklearnTrainingMultiLayerPerceptron.png new file mode 100644 index 0000000000..ebd38d1d56 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingMultiLayerPerceptron.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingMultinomialNaiveBayes.png b/core/gui/src/assets/operator_images/SklearnTrainingMultinomialNaiveBayes.png new file mode 100644 index 0000000000..8de675ba7f Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingMultinomialNaiveBayes.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingNearestCentroid.png b/core/gui/src/assets/operator_images/SklearnTrainingNearestCentroid.png new file mode 100644 index 0000000000..b548043784 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingNearestCentroid.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingPassiveAggressive.png b/core/gui/src/assets/operator_images/SklearnTrainingPassiveAggressive.png new file mode 100644 index 0000000000..1b41270fd0 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingPassiveAggressive.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingPerceptron.png b/core/gui/src/assets/operator_images/SklearnTrainingPerceptron.png new file mode 100644 index 0000000000..858c506282 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingPerceptron.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingProbabilityCalibration.png b/core/gui/src/assets/operator_images/SklearnTrainingProbabilityCalibration.png new file mode 100644 index 0000000000..114dfbac06 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingProbabilityCalibration.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingRandomForest.png b/core/gui/src/assets/operator_images/SklearnTrainingRandomForest.png new file mode 100644 index 0000000000..2ba59197dc Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingRandomForest.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingRidge.png b/core/gui/src/assets/operator_images/SklearnTrainingRidge.png new file mode 100644 index 0000000000..0c1f47c743 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingRidge.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingRidgeCV.png b/core/gui/src/assets/operator_images/SklearnTrainingRidgeCV.png new file mode 100644 index 0000000000..b28c68ee08 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingRidgeCV.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingSDG.png b/core/gui/src/assets/operator_images/SklearnTrainingSDG.png new file mode 100644 index 0000000000..fddaa7bfd0 Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingSDG.png differ diff --git a/core/gui/src/assets/operator_images/SklearnTrainingSVM.png b/core/gui/src/assets/operator_images/SklearnTrainingSVM.png new file mode 100644 index 0000000000..4fb5d4aede Binary files /dev/null and b/core/gui/src/assets/operator_images/SklearnTrainingSVM.png differ diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala index 6aa57f2574..790e463898 100644 --- a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala @@ -76,7 +76,10 @@ import edu.uci.ics.amber.operator.sklearn.training.{ SklearnTrainingGaussianNaiveBayesOpDesc, SklearnTrainingGradientBoostingOpDesc, SklearnTrainingKNNOpDesc, + SklearnTrainingLinearRegressionOpDesc, SklearnTrainingLinearSVMOpDesc, + SklearnTrainingLogisticRegressionCVOpDesc, + SklearnTrainingLogisticRegressionOpDesc, SklearnTrainingMultiLayerPerceptronOpDesc, SklearnTrainingMultinomialNaiveBayesOpDesc, SklearnTrainingNearestCentroidOpDesc, @@ -273,61 +276,79 @@ trait StateTransferFunc value = classOf[SklearnLogisticRegressionCVOpDesc], name = "SklearnLogisticRegressionCV" ), - new Type(value = classOf[SklearnTrainingRidgeOpDesc], name = "SklearnRidge"), - new Type(value = classOf[SklearnTrainingRidgeCVOpDesc], name = "SklearnRidgeCV"), - new Type(value = classOf[SklearnTrainingSDGOpDesc], name = "SklearnSDG"), + new Type(value = classOf[SklearnTrainingRidgeOpDesc], name = "SklearnTrainingRidge"), + new Type(value = classOf[SklearnTrainingRidgeCVOpDesc], name = "SklearnTrainingRidgeCV"), + new Type(value = classOf[SklearnTrainingSDGOpDesc], name = "SklearnTrainingSDG"), new Type( value = classOf[SklearnTrainingPassiveAggressiveOpDesc], - name = "SklearnPassiveAggressive" + name = "SklearnTrainingPassiveAggressive" ), - new Type(value = classOf[SklearnTrainingPerceptronOpDesc], name = "SklearnPerceptron"), - new Type(value = classOf[SklearnTrainingKNNOpDesc], name = "SklearnKNN"), + new Type(value = classOf[SklearnTrainingPerceptronOpDesc], name = "SklearnTrainingPerceptron"), + new Type(value = classOf[SklearnTrainingKNNOpDesc], name = "SklearnTrainingKNN"), new Type( value = classOf[SklearnTrainingNearestCentroidOpDesc], - name = "SklearnNearestCentroid" + name = "SklearnTrainingNearestCentroid" ), - new Type(value = classOf[SklearnTrainingSVMOpDesc], name = "SklearnSVM"), - new Type(value = classOf[SklearnTrainingLinearSVMOpDesc], name = "SklearnLinearSVM"), - new Type(value = classOf[SklearnTrainingDecisionTreeOpDesc], name = "SklearnDecisionTree"), - new Type(value = classOf[SklearnTrainingExtraTreeOpDesc], name = "SklearnExtraTree"), + new Type(value = classOf[SklearnTrainingSVMOpDesc], name = "SklearnTrainingSVM"), + new Type(value = classOf[SklearnTrainingLinearSVMOpDesc], name = "SklearnTrainingLinearSVM"), + new Type( + value = classOf[SklearnTrainingDecisionTreeOpDesc], + name = "SklearnTrainingDecisionTree" + ), + new Type(value = classOf[SklearnTrainingExtraTreeOpDesc], name = "SklearnTrainingExtraTree"), new Type( value = classOf[SklearnTrainingMultiLayerPerceptronOpDesc], - name = "SklearnMultiLayerPerceptron" + name = "SklearnTrainingMultiLayerPerceptron" ), new Type( value = classOf[SklearnTrainingProbabilityCalibrationOpDesc], - name = "SklearnProbabilityCalibration" + name = "SklearnTrainingProbabilityCalibration" ), - new Type(value = classOf[SklearnTrainingRandomForestOpDesc], name = "SklearnRandomForest"), - new Type(value = classOf[SklearnTrainingBaggingOpDesc], name = "SklearnBagging"), + new Type( + value = classOf[SklearnTrainingRandomForestOpDesc], + name = "SklearnTrainingRandomForest" + ), + new Type(value = classOf[SklearnTrainingBaggingOpDesc], name = "SklearnTrainingBagging"), new Type( value = classOf[SklearnTrainingGradientBoostingOpDesc], - name = "SklearnGradientBoosting" + name = "SklearnTrainingGradientBoosting" ), new Type( value = classOf[SklearnTrainingAdaptiveBoostingOpDesc], - name = "SklearnAdaptiveBoosting" + name = "SklearnTrainingAdaptiveBoosting" ), - new Type(value = classOf[SklearnTrainingExtraTreesOpDesc], name = "SklearnExtraTrees"), + new Type(value = classOf[SklearnTrainingExtraTreesOpDesc], name = "SklearnTrainingExtraTrees"), new Type( value = classOf[SklearnTrainingGaussianNaiveBayesOpDesc], - name = "SklearnGaussianNaiveBayes" + name = "SklearnTrainingGaussianNaiveBayes" ), new Type( value = classOf[SklearnTrainingMultinomialNaiveBayesOpDesc], - name = "SklearnMultinomialNaiveBayes" + name = "SklearnTrainingMultinomialNaiveBayes" ), new Type( value = classOf[SklearnTrainingComplementNaiveBayesOpDesc], - name = "SklearnComplementNaiveBayes" + name = "SklearnTrainingComplementNaiveBayes" ), new Type( value = classOf[SklearnTrainingBernoulliNaiveBayesOpDesc], - name = "SklearnBernoulliNaiveBayes" + name = "SklearnTrainingBernoulliNaiveBayes" ), new Type( value = classOf[SklearnTrainingDummyClassifierOpDesc], - name = "SklearnDummyClassifier" + name = "SklearnTrainingDummyClassifier" + ), + new Type( + value = classOf[SklearnTrainingLinearRegressionOpDesc], + name = "SklearnTrainingLinearRegression" + ), + new Type( + value = classOf[SklearnTrainingLogisticRegressionOpDesc], + name = "SklearnTrainingLogisticRegression" + ), + new Type( + value = classOf[SklearnTrainingLogisticRegressionCVOpDesc], + name = "SklearnTrainingLogisticRegressionCV" ), new Type(value = classOf[SklearnLogisticRegressionOpDesc], name = "SklearnLogisticRegression"), new Type( diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDesc.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDesc.scala new file mode 100644 index 0000000000..12d5eddb87 --- /dev/null +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDesc.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package edu.uci.ics.amber.operator.sklearn.training + +class SklearnTrainingLinearRegressionOpDesc extends SklearnTrainingOpDesc { + override def getImportStatements = "from sklearn.linear_model import LinearRegression" + override def getUserFriendlyModelName = "Training: Linear Regression" +}
