[
https://issues.apache.org/jira/browse/BEAM-13562?focusedWorklogId=711923&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-711923
]
ASF GitHub Bot logged work on BEAM-13562:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 20/Jan/22 08:50
Start Date: 20/Jan/22 08:50
Worklog Time Spent: 10m
Work Description: daria-malkova commented on a change in pull request
#16563:
URL: https://github.com/apache/beam/pull/16563#discussion_r788517325
##########
File path:
playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
##########
@@ -65,6 +70,11 @@ func Setup(sdk pb.Sdk, code string, pipelineId uuid.UUID,
workingDir, pipelinesF
lc.DeleteFolders()
return nil, errors.New("error during create necessary
files for the Java sdk")
}
+ case pb.Sdk_SDK_SCIO:
+ if lc, err = prepareSbtFiles(lc,
lc.Paths.AbsoluteBaseFolderPath, workingDir); err != nil {
+ lc.DeleteFolders()
+ return nil, errors.New("error during create necessary
files for the Java sdk")
Review comment:
Done
##########
File path:
playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
##########
@@ -143,3 +153,42 @@ func updateJavaLogConfigFile(paths fs_tool.LifeCyclePaths)
error {
}
return nil
}
+
+func prepareSbtFiles(lc *fs_tool.LifeCycle, pipelineFolder string, workingDir
string) (*fs_tool.LifeCycle, error) {
+ cmd := exec.Command("sh", filepath.Join(workingDir,
"new_scio_project.sh"))
+ cmd.Dir = pipelineFolder
+ fmt.Println(cmd.String())
Review comment:
Done
##########
File path: playground/backend/internal/validators/scio_validators.go
##########
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package validators
+
+import (
+ "beam.apache.org/playground/backend/internal/fs_tool"
+)
+
+const (
+ scalaExtension = ".scala"
+ //scioUnitTestPattern = "@Test"
Review comment:
Done
##########
File path: playground/backend/new_scio_project.sh
##########
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+yes scioproject | sbt new spotify/scio-template.g8
Review comment:
Done
##########
File path:
playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
##########
@@ -143,3 +153,42 @@ func updateJavaLogConfigFile(paths fs_tool.LifeCyclePaths)
error {
}
return nil
}
+
+func prepareSbtFiles(lc *fs_tool.LifeCycle, pipelineFolder string, workingDir
string) (*fs_tool.LifeCycle, error) {
+ cmd := exec.Command("sh", filepath.Join(workingDir,
"new_scio_project.sh"))
+ cmd.Dir = pipelineFolder
+ fmt.Println(cmd.String())
+ _, err := cmd.Output()
+ if err != nil {
+ return nil, err
+ }
+
+ sourceFileFolder := filepath.Join(pipelineFolder, projectPath)
+ fileName := lc.Paths.SourceFileName
+ absFileFolderPath, _ := filepath.Abs(sourceFileFolder)
+ absFilePath, _ := filepath.Abs(filepath.Join(absFileFolderPath,
fileName))
+ absLogFilePath, _ := filepath.Abs(filepath.Join(absFileFolderPath,
logFileName))
+ projectFolder, _ := filepath.Abs(filepath.Join(pipelineFolder,
scioProjectName))
+ executableName := lc.Paths.ExecutableName
+
+ _, err = exec.Command("rm", filepath.Join(absFileFolderPath,
"WordCount.scala")).Output()
+ if err != nil {
+ return nil, err
+ }
Review comment:
and it is better to delete it to avoid conflicts
##########
File path:
playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
##########
@@ -143,3 +153,42 @@ func updateJavaLogConfigFile(paths fs_tool.LifeCyclePaths)
error {
}
return nil
}
+
+func prepareSbtFiles(lc *fs_tool.LifeCycle, pipelineFolder string, workingDir
string) (*fs_tool.LifeCycle, error) {
+ cmd := exec.Command("sh", filepath.Join(workingDir,
"new_scio_project.sh"))
+ cmd.Dir = pipelineFolder
+ fmt.Println(cmd.String())
+ _, err := cmd.Output()
+ if err != nil {
+ return nil, err
+ }
+
+ sourceFileFolder := filepath.Join(pipelineFolder, projectPath)
+ fileName := lc.Paths.SourceFileName
+ absFileFolderPath, _ := filepath.Abs(sourceFileFolder)
+ absFilePath, _ := filepath.Abs(filepath.Join(absFileFolderPath,
fileName))
+ absLogFilePath, _ := filepath.Abs(filepath.Join(absFileFolderPath,
logFileName))
+ projectFolder, _ := filepath.Abs(filepath.Join(pipelineFolder,
scioProjectName))
+ executableName := lc.Paths.ExecutableName
+
+ _, err = exec.Command("rm", filepath.Join(absFileFolderPath,
"WordCount.scala")).Output()
+ if err != nil {
+ return nil, err
+ }
Review comment:
we have a template to create scio sbt project that is automatically add
WordCount example to the created project
##########
File path: playground/backend/internal/validators/scio_validators.go
##########
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package validators
+
+import (
+ "beam.apache.org/playground/backend/internal/fs_tool"
+)
+
+const (
+ scalaExtension = ".scala"
+ //scioUnitTestPattern = "@Test"
+)
+
+// GetScioValidators return validators methods that should be applied to scio
code
+// The last validator should check that the code is unit tests or not
+func GetScioValidators(filePath string) *[]Validator {
+ validatorArgs := make([]interface{}, 2)
+ validatorArgs[0] = filePath
+ validatorArgs[1] = scalaExtension
+ pathCheckerValidator := Validator{
+ Validator: fs_tool.CheckPathIsValid,
+ Args: validatorArgs,
+ Name: "Valid path",
+ }
+ unitTestValidator := Validator{
+ Validator: checkIsUnitTestScio,
+ Args: validatorArgs,
+ Name: UnitTestValidatorName,
+ }
+ validators := []Validator{pathCheckerValidator, unitTestValidator}
+ return &validators
+}
+
+//checkIsUnitTestScio checks if the pipeline is a UnitTest
+func checkIsUnitTestScio(args ...interface{}) (bool, error) {
+ return false, nil
Review comment:
Right
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@beam.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 711923)
Time Spent: 2h (was: 1h 50m)
> [Playground] Support SCIO SDK
> -----------------------------
>
> Key: BEAM-13562
> URL: https://issues.apache.org/jira/browse/BEAM-13562
> Project: Beam
> Issue Type: Improvement
> Components: beam-playground
> Reporter: Ilya Kozyrev
> Assignee: Daria Malkova
> Priority: P2
> Labels: beam-playground-backend, beam-playground-frontend,
> beam-playground-sprint-7
> Time Spent: 2h
> Remaining Estimate: 0h
>
> As a Beam Playground {*}user{*}, I want to get execution output for the
> specific SCIO example code that I run in Beam Playground
> Acceptance criteria:
> # Beam SCIO example code executed on the backend
> # Output received by the frontend
> # Compilation failures are caught and redirected to the frontend app
> # Runtime failures are caught and redirected to the frontend app
>
> _{*}Scenario{*}: SCIO example code running_
> _{*}Given{*}: User ran valid Apache Beam Python example code in Beam
> Playground_
> _{*}When{*}: Backend accepts the gRPC request_
> _{*}Then{*}: Backend apply preparation steps for SCIO SDK_
> _*AND:* Backend apply validation steps for SCIO SDK_
> _*AND:* Backend compile scala code in jar file using sbt_
> _*AND:* Backend runs a new background process using the java console command
> to run jar file_
>
> __
> _{*}Given{*}: Backend app runs jar file using java console command_
> _{*}When{*}: Code executed completely_
> _{*}And{*}: Stdout and stderr outputs located at local fs_
> _{*}Then{*}: Backend reads the stdout and sends its content to the frontend
> with complete message code_
> __
> _{*}Given{*}: Backend app runs jar file using java console command_
> _{*}When{*}: Code executed with error_
> _{*}And{*}: Stdout and stderr outputs located at local fs_
> _{*}Then{*}: Backend reads the stderr and sends its content to the frontend
> with an error message code_
> __
> _{*}Given{*}: Backend app runs executable file using java console command_
> _{*}When{*}: Execution is finished with any code_
> _{*}And{*}: Backend sent a response to frontend_
> _{*}Then{*}: Backend clean-ups build and runtime artifacts._
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
