Thespica commented on code in PR #616: URL: https://github.com/apache/incubator-graphar/pull/616#discussion_r1818000466
########## cli/src/main.cc: ########## @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <pybind11/pybind11.h> +#include <pybind11/stl.h> + +#include <graphar/filesystem.h> +#include <graphar/graph_info.h> +#include <graphar/reader_util.h> +#include "importer.h" + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + +std::string ShowGraph(const std::string& path) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + return graph_info->Dump().value(); +} + +std::string ShowVertex(const std::string& path, + const std::string& vertex_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto vertex_info = graph_info->GetVertexInfo(vertex_type); + return vertex_info->Dump().value(); +} + +std::string ShowEdge(const std::string& path, const std::string& src_type, + const std::string& edge_type, + const std::string& dst_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto edge_info = graph_info->GetEdgeInfo(src_type, edge_type, dst_type); + return edge_info->Dump().value(); +} + +bool CheckGraph(const std::string& path) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + return graph_info->IsValidated(); +} + +bool CheckVertex(const std::string& path, const std::string& vertex_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto vertex_info = graph_info->GetVertexInfo(vertex_type); + return vertex_info->IsValidated(); +} + +bool CheckEdge(const std::string& path, const std::string& src_type, + const std::string& edge_type, const std::string& dst_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto edge_info = graph_info->GetEdgeInfo(src_type, edge_type, dst_type); + return edge_info->IsValidated(); +} + +int64_t GetVertexCount(const std::string& path, + const std::string& vertex_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto graph_prefix = graph_info->GetPrefix(); + auto vertex_info = graph_info->GetVertexInfo(vertex_type); + return graphar::util::GetVertexNum(graph_prefix, vertex_info).value(); +} + +// TODO(ljj): Add this to graphar library + +std::vector<graphar::AdjListType> _GetAdjListTypes( + const std::shared_ptr<graphar::EdgeInfo>& edge_info) { + std::vector<graphar::AdjListType> adj_list_types; + if (edge_info->HasAdjacentListType(graphar::AdjListType::ordered_by_dest)) { + adj_list_types.push_back(graphar::AdjListType::ordered_by_dest); + } + if (edge_info->HasAdjacentListType(graphar::AdjListType::ordered_by_source)) { + adj_list_types.push_back(graphar::AdjListType::ordered_by_source); + } + if (edge_info->HasAdjacentListType(graphar::AdjListType::unordered_by_dest)) { + adj_list_types.push_back(graphar::AdjListType::unordered_by_dest); + } + if (edge_info->HasAdjacentListType( + graphar::AdjListType::unordered_by_source)) { + adj_list_types.push_back(graphar::AdjListType::unordered_by_source); + } + if (adj_list_types.empty()) { + throw std::runtime_error("No valid adj list type found"); + } + return adj_list_types; +} + +int64_t GetEdgeCount(const std::string& path, const std::string& src_type, + const std::string& edge_type, + const std::string& dst_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto graph_prefix = graph_info->GetPrefix(); + auto edge_info = graph_info->GetEdgeInfo(src_type, edge_type, dst_type); + auto adj_list_types = _GetAdjListTypes(edge_info); + auto adj_list_type = adj_list_types[0]; + auto vertices_num_file_path = + edge_info->GetVerticesNumFilePath(adj_list_type).value(); + std::string base_dir; + auto fs = graphar::FileSystemFromUriOrPath(graph_prefix, &base_dir).value(); + std::string vertices_num_path = base_dir + vertices_num_file_path; + auto vertices_num = fs->ReadFileToValue<int64_t>(vertices_num_path).value(); + int max_chunk_index = (vertices_num + edge_info->GetSrcChunkSize() - 1) / + edge_info->GetSrcChunkSize(); + int64_t edge_count = 0; + for (int i = 0; i < max_chunk_index; i++) { + // TODO: file may not exist + edge_count += + graphar::util::GetEdgeNum(graph_prefix, edge_info, adj_list_type, i) + .value(); + } + return edge_count; +} + +std::vector<std::string> GetVertexTypes(const std::string& path) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto vertex_infos = graph_info->GetVertexInfos(); + std::vector<std::string> vertex_types; + for (const auto& vertex_info : vertex_infos) { + vertex_types.push_back(vertex_info->GetType()); + } + return vertex_types; +} + +std::vector<std::vector<std::string>> GetEdgeTypes(const std::string& path) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto edge_infos = graph_info->GetEdgeInfos(); + std::vector<std::vector<std::string>> edge_types; Review Comment: maybe using constructor with vector size will be better. ```suggestion std::vector<std::vector<std::string>> edge_types(edge_infos.size()); ``` ########## cli/src/main.cc: ########## @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <pybind11/pybind11.h> +#include <pybind11/stl.h> + +#include <graphar/filesystem.h> +#include <graphar/graph_info.h> +#include <graphar/reader_util.h> +#include "importer.h" + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + +std::string ShowGraph(const std::string& path) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + return graph_info->Dump().value(); +} + +std::string ShowVertex(const std::string& path, + const std::string& vertex_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto vertex_info = graph_info->GetVertexInfo(vertex_type); + return vertex_info->Dump().value(); +} + +std::string ShowEdge(const std::string& path, const std::string& src_type, + const std::string& edge_type, + const std::string& dst_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto edge_info = graph_info->GetEdgeInfo(src_type, edge_type, dst_type); + return edge_info->Dump().value(); +} + +bool CheckGraph(const std::string& path) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + return graph_info->IsValidated(); +} + +bool CheckVertex(const std::string& path, const std::string& vertex_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto vertex_info = graph_info->GetVertexInfo(vertex_type); + return vertex_info->IsValidated(); +} + +bool CheckEdge(const std::string& path, const std::string& src_type, + const std::string& edge_type, const std::string& dst_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto edge_info = graph_info->GetEdgeInfo(src_type, edge_type, dst_type); + return edge_info->IsValidated(); +} + +int64_t GetVertexCount(const std::string& path, + const std::string& vertex_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto graph_prefix = graph_info->GetPrefix(); + auto vertex_info = graph_info->GetVertexInfo(vertex_type); + return graphar::util::GetVertexNum(graph_prefix, vertex_info).value(); +} + +// TODO(ljj): Add this to graphar library + +std::vector<graphar::AdjListType> _GetAdjListTypes( + const std::shared_ptr<graphar::EdgeInfo>& edge_info) { + std::vector<graphar::AdjListType> adj_list_types; + if (edge_info->HasAdjacentListType(graphar::AdjListType::ordered_by_dest)) { + adj_list_types.push_back(graphar::AdjListType::ordered_by_dest); + } + if (edge_info->HasAdjacentListType(graphar::AdjListType::ordered_by_source)) { + adj_list_types.push_back(graphar::AdjListType::ordered_by_source); + } + if (edge_info->HasAdjacentListType(graphar::AdjListType::unordered_by_dest)) { + adj_list_types.push_back(graphar::AdjListType::unordered_by_dest); + } + if (edge_info->HasAdjacentListType( + graphar::AdjListType::unordered_by_source)) { + adj_list_types.push_back(graphar::AdjListType::unordered_by_source); + } + if (adj_list_types.empty()) { + throw std::runtime_error("No valid adj list type found"); + } + return adj_list_types; +} + +int64_t GetEdgeCount(const std::string& path, const std::string& src_type, + const std::string& edge_type, + const std::string& dst_type) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto graph_prefix = graph_info->GetPrefix(); + auto edge_info = graph_info->GetEdgeInfo(src_type, edge_type, dst_type); + auto adj_list_types = _GetAdjListTypes(edge_info); + auto adj_list_type = adj_list_types[0]; + auto vertices_num_file_path = + edge_info->GetVerticesNumFilePath(adj_list_type).value(); + std::string base_dir; + auto fs = graphar::FileSystemFromUriOrPath(graph_prefix, &base_dir).value(); + std::string vertices_num_path = base_dir + vertices_num_file_path; + auto vertices_num = fs->ReadFileToValue<int64_t>(vertices_num_path).value(); + int max_chunk_index = (vertices_num + edge_info->GetSrcChunkSize() - 1) / + edge_info->GetSrcChunkSize(); + int64_t edge_count = 0; + for (int i = 0; i < max_chunk_index; i++) { + // TODO: file may not exist + edge_count += + graphar::util::GetEdgeNum(graph_prefix, edge_info, adj_list_type, i) + .value(); + } + return edge_count; +} + +std::vector<std::string> GetVertexTypes(const std::string& path) { + auto graph_info = graphar::GraphInfo::Load(path).value(); + auto vertex_infos = graph_info->GetVertexInfos(); + std::vector<std::string> vertex_types; Review Comment: maybe using constructor with vector size will be better. ```suggestion std::vector<std::string> vertex_types(vertex_infos.size()); ``` ########## cli/README.md: ########## @@ -0,0 +1,88 @@ +# GraphAr Cli + +GraphAr Cli uses [pybind11][] and [scikit-build-core][] to bind C++ code into Python and build command line tools through Python. Command line tools developed using [typer][]. + +[pybind11]: https://pybind11.readthedocs.io +[scikit-build-core]: https://scikit-build-core.readthedocs.io +[typer]: https://typer.tiangolo.com/ + +## Requirements + +- Linux (work fine on Ubuntu 22.04) +- Cmake >= 3.15 Review Comment: This is not as same as the requirement in cmake file: `cmake_minimum_required(VERSION 3.15...3.27)` make them aligned ########## cli/src/main.cc: ########## @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <pybind11/pybind11.h> +#include <pybind11/stl.h> + +#include <graphar/filesystem.h> +#include <graphar/graph_info.h> +#include <graphar/reader_util.h> +#include "importer.h" + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + +std::string ShowGraph(const std::string& path) { + auto graph_info = graphar::GraphInfo::Load(path).value(); Review Comment: the returned object with `Result` wrapped should be checked before getting its value /cc @acezen ########## .github/workflows/cli.yml: ########## @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: GraphAr CLI CI + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + paths: + - 'cpp/**' + - 'cli/**' + - '.github/workflows/ci.yml' + - '.github/workflows/cli.yml' + pull_request: + branches: + - main + paths: + - 'cpp/**' + - 'cli/**' + - '.github/workflows/ci.yml' + - '.github/workflows/cli.yml' +concurrency: + group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + ubuntu: + name: Ubuntu 22.04 CLI + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') && !github.event.pull_request.draft }} + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Install dependencies + run: | + + # install the latest arrow deb to test arrow + wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ + -P /tmp/ + sudo apt-get install -y /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb + sudo apt-get update -y + sudo apt install -y libarrow-dev \ + libarrow-dataset-dev \ + libarrow-acero-dev \ + libparquet-dev + sudo apt-get install -y ccache libcurl4-openssl-dev + + - name: Install GraphAr CLI and Run Tests + working-directory: "cli" + run: | + pip install ./ -v + graphar-cli --help + graphar-cli check -f ../testing/neo4j/MovieGraph.graph.yml + graphar-cli show -f ../testing/neo4j/MovieGraph.graph.yml -v Person + graphar-cli show -f ../testing/neo4j/MovieGraph.graph.yml -es Person -e ACTED_IN -ed Movie + graphar-cli import -c import.mini.yml + + - name: Upload coverage reports to Codecov + uses: codecov/[email protected] + with: + token: ${{ secrets.CODECOV_TOKEN }} + + macos: + name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} CLI + runs-on: macos-${{ matrix.macos-version }} + if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.event.pull_request.draft == false }} + strategy: + fail-fast: false + matrix: + include: + - architecture: AMD64 + macos-version: "12" + - architecture: ARM64 + macos-version: "14" + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Install dependencies + run: | + brew bundle --file=cpp/Brewfile + + + - name: Build GraphAr And Run Tests + working-directory: "cli" + run: | + pip install ./ + graphar-cli --help + graphar-cli check -f ../testing/neo4j/MovieGraph.graph.yml + graphar-cli show -f ../testing/neo4j/MovieGraph.graph.yml -v Person + graphar-cli show -f ../testing/neo4j/MovieGraph.graph.yml -es Person -e ACTED_IN -ed Movie + graphar-cli import -c import.mini.yml Review Comment: Is there a command for test like `pytest` or other single command, rather than test one by one? If the answer is yes, and will do it later, adding a TODO tag here please. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
