This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8aa9faa [chore](docker) Add docker dev image with ldb-toolchain
(#7838)
8aa9faa is described below
commit 8aa9faa7cb4f91f1fec3202cfb5340723a620548
Author: Mingyu Chen <morningman....@gmail.com>
AuthorDate: Mon Jan 24 21:12:15 2022 +0800
[chore](docker) Add docker dev image with ldb-toolchain (#7838)
Add docker images `apache/incubator-doris:build-env-ldb-toolchain-latest`,
which is built with ldb-toolchain
---
docker/Dockerfile | 91 +++++-----------------
docker/{Dockerfile => Dockerfile.gcc10} | 3 +-
docker/Dockerfile.gcc7 | 1 +
docker/README.md | 4 +-
.../installing/compilation-with-ldb-toolchain.md | 4 +-
docs/en/installing/compilation.md | 15 ++--
.../installing/compilation-with-ldb-toolchain.md | 4 +-
docs/zh-CN/installing/compilation.md | 15 ++--
.../src/main/java/org/apache/doris/load/Load.java | 4 +
9 files changed, 52 insertions(+), 89 deletions(-)
diff --git a/docker/Dockerfile b/docker/Dockerfile
index f0b7b92..207469e 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -18,67 +18,34 @@
FROM centos:7 AS builder
# install dependencies
-RUN yum makecache && yum -y update && yum -y groupinstall 'Development Tools'
&& \
- yum install -y byacc automake java-11-openjdk-devel
java-1.8.0-openjdk-devel libtool bison binutils-devel zip \
- unzip ncurses-devel curl git wget python2 glibc-static
java-1.8.0-openjdk-devel ccache \
- libstdc++-static which psl libpsl-devel centos-release-scl && \
- yum install -y devtoolset-10 devtoolset-10-gcc
devtoolset-10-libubsan-devel devtoolset-10-liblsan-devel \
- devtoolset-10-libasan-devel
-
-# build cmake
-ARG CMAKE_VERSION=3.22.1
-ARG
CMAKE_BASE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}
-RUN wget ${CMAKE_BASE_URL}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh -q -O
/tmp/cmake-install.sh && \
- chmod u+x /tmp/cmake-install.sh && \
- /tmp/cmake-install.sh --skip-license --prefix=/usr --exclude-subdir && \
- rm /tmp/cmake-install.sh
-
-# build ninja
-ARG NINJA_VER=1.10.2
-ARG
NINJA_BASE_URL=https://github.com/ninja-build/ninja/releases/download/v${NINJA_VER}
-RUN wget -q ${NINJA_BASE_URL}/ninja-linux.zip -O /tmp/ninja-linux.zip && \
- unzip /tmp/ninja-linux.zip -d /usr/bin/ && \
- rm /tmp/ninja-linux.zip
+RUN yum install -y byacc patch automake libtool make which file ncurses-devel
gettext-devel unzip bzip2 zip util-linux wget git python2 \
+ bison java-11-openjdk-devel java-1.8.0-openjdk-devel
# install maven 3.6.3
-ARG MAVEN_VERSION=3.6.3
-ARG
MAVEN_BASE_URL=https://downloads.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries
RUN mkdir -p /usr/share/maven /usr/share/maven/ref && \
- wget -q -O /tmp/apache-maven.tar.gz
${MAVEN_BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz && \
+ wget -q -O /tmp/apache-maven.tar.gz
https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/apache-maven-3.6.3-bin.tar.gz
&& \
tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1
&& \
rm -f /tmp/apache-maven.tar.gz && \
ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
-# build flex
-ARG FLEX_VERSION=2.6.4
-RUN wget
https://github.com/westes/flex/releases/download/v$FLEX_VERSION/flex-$FLEX_VERSION.tar.gz
\
- -q -O /tmp/flex-$FLEX_VERSION.tar.gz \
- && cd /tmp/ \
- && tar -xf flex-$FLEX_VERSION.tar.gz \
- && cd flex-$FLEX_VERSION \
- && ./configure --enable-shared=NO \
- && make \
- && make install \
- && rm /tmp/flex-$FLEX_VERSION.tar.gz \
- && rm -rf /tmp/flex-$FLEX_VERSION
-
# install nodejs
-ARG NODEJS_VERSION=14.16.0
-RUN wget
https://nodejs.org/dist/v$NODEJS_VERSION/node-v$NODEJS_VERSION-linux-x64.tar.gz
\
- -q -O /tmp/node-v$NODEJS_VERSION-linux-x64.tar.gz \
- && cd /tmp/ && tar -xf node-v$NODEJS_VERSION-linux-x64.tar.gz \
- && cp -r node-v$NODEJS_VERSION-linux-x64/* /usr/local/ \
- && rm /tmp/node-v$NODEJS_VERSION-linux-x64.tar.gz && rm -rf
node-v$NODEJS_VERSION-linux-x64
-
-
-ENV BASH_ENV=/opt/rh/devtoolset-10/enable \
- ENV=/opt/rh/devtoolset-10/enable \
- PROMPT_COMMAND=". /opt/rh/devtoolset-10/enable"
+RUN wget
https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v12.13.0-linux-x64.tar.gz
\
+ -q -O /tmp/node-v12.13.0-linux-x64.tar.gz \
+ && cd /tmp/ && tar -xf node-v12.13.0-linux-x64.tar.gz \
+ && cp -r node-v12.13.0-linux-x64/* /usr/local/ \
+ && rm /tmp/node-v12.13.0-linux-x64.tar.gz && rm -rf node-v12.13.0-linux-x64
+
+# install ldb-toolchain
+RUN wget
https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/ldb_toolchain_gen-v0.4.sh
\
+ -q -O /tmp/ldb_toolchain_gen.sh \
+ && sh /tmp/ldb_toolchain_gen.sh /var/local/ldb-toolchain/ \
+ && rm /tmp/ldb_toolchain_gen.sh
# there is a repo which is included all of thirdparty
-ENV REPOSITORY_URL=https://doris-thirdparty-repo.bj.bcebos.com/thirdparty
-ENV DEFAULT_DIR=/var/local
-ENV JAVA_HOME="/usr/lib/jvm/java-11"
+ENV REPOSITORY_URL="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/"; \
+ DEFAULT_DIR="/var/local" \
+ JAVA_HOME="/usr/lib/jvm/java-11" \
+ PATH="/var/local/ldb-toolchain/bin/:$PATH"
# clone lastest source code, download and build third party
COPY incubator-doris ${DEFAULT_DIR}/incubator-doris
@@ -92,32 +59,16 @@ RUN cd ${DEFAULT_DIR}/incubator-doris && /bin/bash
thirdparty/build-thirdparty.s
RUN alternatives --set java java-11-openjdk.x86_64 && alternatives --set javac
java-11-openjdk.x86_64
-# build clang 11
-RUN wget -P ${DEFAULT_DIR}
https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-project-11.0.0.tar.xz
\
- && tar Jxf ${DEFAULT_DIR}/llvm-project-11.0.0.tar.xz -C ${DEFAULT_DIR} \
- && echo "source /opt/rh/devtoolset-10/enable" >
${DEFAULT_DIR}/build_clang.sh \
- && echo "cd ${DEFAULT_DIR}/llvm-project-11.0.0" >>
${DEFAULT_DIR}/build_clang.sh \
- && echo "mkdir build" >> ${DEFAULT_DIR}/build_clang.sh \
- && echo "cd build" >> ${DEFAULT_DIR}/build_clang.sh \
- && echo "cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RTTI=ON
-DLLVM_ENABLE_PROJECTS=\"clang;libcxx;libcxxabi\" -G \"Unix Makefiles\" ../llvm
" >> ${DEFAULT_DIR}/build_clang.sh \
- && echo "make -j $(nproc)" >> ${DEFAULT_DIR}/build_clang.sh \
- && echo "make install" >> ${DEFAULT_DIR}/build_clang.sh \
- && echo "rm -rf ${DEFAULT_DIR}/llvm-project-11.0.0*" >>
${DEFAULT_DIR}/build_clang.sh \
- && chmod +x ${DEFAULT_DIR}/build_clang.sh \
- && ${DEFAULT_DIR}/build_clang.sh
-
FROM scratch
COPY --from=builder / /
ENV JAVA_HOME="/usr/lib/jvm/java-11" \
MAVEN_HOME="/usr/share/maven" \
- BASH_ENV="/opt/rh/devtoolset-10/enable" \
- ENV="/opt/rh/devtoolset-10/enable" \
- PROMPT_COMMAND=". /opt/rh/devtoolset-10/enable" \
REPOSITORY_URL="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty"; \
DEFAULT_DIR="/var/local" \
- PATH="/var/local/thirdparty/installed/bin:$PATH" \
+
PATH="/var/local/ldb-toolchain/bin/:/var/local/thirdparty/installed/bin/:$PATH"
\
DORIS_THIRDPARTY="/var/local/thirdparty"
WORKDIR /root
-CMD ["/bin/bash"]
\ No newline at end of file
+CMD ["/bin/bash"]
+
diff --git a/docker/Dockerfile b/docker/Dockerfile.gcc10
similarity index 99%
copy from docker/Dockerfile
copy to docker/Dockerfile.gcc10
index f0b7b92..777c4e0 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile.gcc10
@@ -120,4 +120,5 @@ ENV JAVA_HOME="/usr/lib/jvm/java-11" \
DORIS_THIRDPARTY="/var/local/thirdparty"
WORKDIR /root
-CMD ["/bin/bash"]
\ No newline at end of file
+CMD ["/bin/bash"]
+
diff --git a/docker/Dockerfile.gcc7 b/docker/Dockerfile.gcc7
index 9b4e6d9..2ed3eb4 100644
--- a/docker/Dockerfile.gcc7
+++ b/docker/Dockerfile.gcc7
@@ -116,3 +116,4 @@ ENV MAVEN_HOME /usr/share/maven
ENV DORIS_THIRDPARTY /var/local/thirdparty
WORKDIR /root
CMD ["/bin/bash"]
+
diff --git a/docker/README.md b/docker/README.md
index 72b5182..93bc1f5 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -73,7 +73,7 @@ used to build this image before)
```console
$ docker run -it -v /your/local/path/incubator-doris/:/root/incubator-doris/
doris:v1.0
-$ $ docker run -it -v /your/local/.m2:/root/.m2 -v
/your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/
doris:v1.0
+$ docker run -it -v /your/local/.m2:/root/.m2 -v
/your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/
doris:v1.0
```
Then you can build source code inside the container.
@@ -97,4 +97,4 @@ The version of jdk you used to run FE must be the same
version you used to compi
### Latest update time
-2021-12-15
+2022-1-23
diff --git a/docs/en/installing/compilation-with-ldb-toolchain.md
b/docs/en/installing/compilation-with-ldb-toolchain.md
index 0faf566..a75f714 100644
--- a/docs/en/installing/compilation-with-ldb-toolchain.md
+++ b/docs/en/installing/compilation-with-ldb-toolchain.md
@@ -28,6 +28,8 @@ under the License.
This document describes how to compile Doris using the LDB toolchain. This
method is currently used as a supplement to the Docker compilation method to
facilitate developers and users without a Docker environment to compile Doris
source code.
+> You can still compile the latest code using the Docker development image:
`apache/incubator-doris:build-env-ldb-toolchain-latest`
+
> Thanks to [Amos Bird](https://github.com/amosbird) for this contribution.
## Prepare the environment
@@ -62,7 +64,7 @@ This works for most Linux distributions (CentOS, Ubuntu,
etc.).
3. Download and install other compiled components
1.
[Java8](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/jdk-8u131-linux-x64.tar.gz)
- 2. [Apache Maven
3.8.4](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/apache-maven-3.6.3-bin.tar.gz)
+ 2. [Apache Maven
3.6.3](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/apache-maven-3.6.3-bin.tar.gz)
3. [Node
v12.13.0](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v12.13.0-linux-x64.tar.gz)
If your environment is somehow minimal, additional packages should be
installed before compiling Doris. The following instructions describe how to
setup a minimal CentOS 6 box to compile Doris. Other linux distros should be
similar.
diff --git a/docs/en/installing/compilation.md
b/docs/en/installing/compilation.md
index a8c6131..f7a2781 100644
--- a/docs/en/installing/compilation.md
+++ b/docs/en/installing/compilation.md
@@ -35,19 +35,19 @@ This document focuses on how to code Doris through source
code.
1. Download Docker Mirror
- `$ docker pull apache/incubator-doris:build-env-latest`
+ `$ docker pull apache/incubator-doris:build-env-ldb-toolchain-latest`
Check mirror download completed:
```
$ docker images
- REPOSITORY TAG IMAGE ID CREATED
SIZE
- apache/incubator-doris build-env-latest 49f68cecbc1a 4 days ago
3.76GB
+ REPOSITORY TAG IMAGE ID
CREATED SIZE
+ apache/incubator-doris build-env-ldb-toolchain-latest 49f68cecbc1a
4 days ago 3.76GB
```
> Note1: For different versions of Doris, you need to download the
> corresponding mirror version. From Apache Doris 0.15 version, the docker
> image will keep same version number with Doris. For example, you can use
> `apache/incubator-doris:build-env-for-0.15.0` to compile Apache Doris 0.15.0.
>
-> Node2: `apache/incubator-doris:build-env-latest` is for compiling trunk
code, and will be updated along with trunk code. View the update time in
`docker/README.md`
+> Node2: `apache/incubator-doris:build-env-ldb-toolchain-latest` is for
compiling trunk code, and will be updated along with trunk code. View the
update time in `docker/README.md`
| image version | commit id | release version |
|---|---|---|
@@ -56,7 +56,8 @@ This document focuses on how to code Doris through source
code.
| apache/incubator-doris:build-env-1.2 |
[4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8)
or later | 0.12.x - 0.14.0 |
| apache/incubator-doris:build-env-1.3.1 |
[ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f)
or later | 0.14.x |
| apache/incubator-doris:build-env-for-0.15.0 |
[a81f4da](https://github.com/apache/incubator-doris/commit/a81f4da4e461a54782a96433b746d07be89e6b54)
or later | 0.15.0 |
-| apache/incubator-doris:build-env-latest | trunk | trunk |
+| apache/incubator-doris:build-env-latest | before
[0efef1b](https://github.com/apache/incubator-doris/commit/0efef1b332300887ee0473f9df9bdd9d7297d824)
| |
+| apache/incubator-doris:build-env-ldb-toolchain-latest | trunk | trunk |
**note**:
@@ -84,14 +85,14 @@ This document focuses on how to code Doris through source
code.
2. Running Mirror
- `$ docker run -it apache/incubator-doris:build-env-latest`
+ `$ docker run -it apache/incubator-doris:build-env-ldb-toolchain-latest`
It is recommended to run the container by mounting the local Doris source
directory, so that the compiled binary file will be stored in the host machine
and will not disappear because the container exits.
At the same time, it is recommended to mount the maven `.m2` directory in
the mirror to the host directory at the same time to prevent repeated
downloading of maven's dependent libraries each time the compilation is started.
```
- $ docker run -it -v /your/local/.m2:/root/.m2 -v
/your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/
apache/incubator-doris:build-env-latest
+ $ docker run -it -v /your/local/.m2:/root/.m2 -v
/your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/
apache/incubator-doris:build-env-ldb-toolchain-latest
```
3. Download source code
diff --git a/docs/zh-CN/installing/compilation-with-ldb-toolchain.md
b/docs/zh-CN/installing/compilation-with-ldb-toolchain.md
index 95e30d8..1829665 100644
--- a/docs/zh-CN/installing/compilation-with-ldb-toolchain.md
+++ b/docs/zh-CN/installing/compilation-with-ldb-toolchain.md
@@ -28,6 +28,8 @@ under the License.
本文档主要介绍如何使用 LDB toolchain 编译 Doris。该方式目前作为 Docker 编译方式的补充，方便没有 Docker
环境的开发者和用户编译 Doris 源码。
+> 您依然可以使用 Docker
开发镜像编译最新代码：`apache/incubator-doris:build-env-ldb-toolchain-latest`
+
> 感谢 [Amos Bird](https://github.com/amosbird) 的贡献。
## 准备编译环境
@@ -62,7 +64,7 @@ under the License.
3. 下载并安装其他编译组件
1.
[Java8](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/jdk-8u131-linux-x64.tar.gz)
- 2. [Apache Maven
3.8.4](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/apache-maven-3.6.3-bin.tar.gz)
+ 2. [Apache Maven
3.6.3](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/apache-maven-3.6.3-bin.tar.gz)
3. [Node
v12.13.0](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v12.13.0-linux-x64.tar.gz)
对于不同的 Linux 发行版，可能默认包含的组件不同。因此可能需要安装一些额外的组件。下面以 centos6 为例，其他发行版类似：
diff --git a/docs/zh-CN/installing/compilation.md
b/docs/zh-CN/installing/compilation.md
index d5b790a..2b0e1bb 100644
--- a/docs/zh-CN/installing/compilation.md
+++ b/docs/zh-CN/installing/compilation.md
@@ -34,19 +34,19 @@ under the License.
1. 下载 Docker 镜像
- `$ docker pull apache/incubator-doris:build-env-latest`
+ `$ docker pull apache/incubator-doris:build-env-ldb-toolchain-latest`
检查镜像下载完成：
```
$ docker images
- REPOSITORY TAG IMAGE ID CREATED
SIZE
- apache/incubator-doris build-env-latest 49f68cecbc1a 4 days ago
3.76GB
+ REPOSITORY TAG IMAGE ID
CREATED SIZE
+ apache/incubator-doris build-env-ldb-toolchain-latest 49f68cecbc1a
4 days ago 3.76GB
```
> 注1：针对不同的 Doris 版本，需要下载对应的镜像版本。从 Apache Doris 0.15 版本起，后续镜像版本号将与 Doris
> 版本号统一。比如可以使用 `apache/incubator-doris:build-env-for-0.15.0 ` 来编译 0.15.0 版本。
>
-> 注2：`apache/incubator-doris:build-env-latest` 用于编译最新主干版本代码，会随主干版本不断更新。可以查看
`docker/README.md` 中的更新时间。
+> 注2：`apache/incubator-doris:build-env-ldb-toolchain-latest`
用于编译最新主干版本代码，会随主干版本不断更新。可以查看 `docker/README.md` 中的更新时间。
| 镜像版本 | commit id | doris 版本 |
|---|---|---|
@@ -55,7 +55,8 @@ under the License.
| apache/incubator-doris:build-env-1.2 |
[4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8)
| 0.12.x - 0.14.0 |
| apache/incubator-doris:build-env-1.3.1 |
[ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f)
| 0.14.x |
| apache/incubator-doris:build-env-for-0.15.0 |
[a81f4da](https://github.com/apache/incubator-doris/commit/a81f4da4e461a54782a96433b746d07be89e6b54)
or later | 0.15.0 |
-| apache/incubator-doris:build-env-latest | trunk | trunk |
+| apache/incubator-doris:build-env-latest | before
[0efef1b](https://github.com/apache/incubator-doris/commit/0efef1b332300887ee0473f9df9bdd9d7297d824)
| |
+| apache/incubator-doris:build-env-ldb-toolchain-latest | trunk | |
**注意**：
@@ -83,14 +84,14 @@ under the License.
2. 运行镜像
- `$ docker run -it apache/incubator-doris:build-env-latest`
+ `$ docker run -it apache/incubator-doris:build-env-ldb-toolchain-latest`
建议以挂载本地 Doris 源码目录的方式运行镜像，这样编译的产出二进制文件会存储在宿主机中，不会因为镜像退出而消失。
同时，建议同时将镜像中 maven 的 `.m2` 目录挂载到宿主机目录，以防止每次启动镜像编译时，重复下载 maven 的依赖库。
```
- $ docker run -it -v /your/local/.m2:/root/.m2 -v
/your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/
apache/incubator-doris:build-env-latest
+ $ docker run -it -v /your/local/.m2:/root/.m2 -v
/your/local/incubator-doris-DORIS-x.x.x-release/:/root/incubator-doris-DORIS-x.x.x-release/
apache/incubator-doris:build-env-ldb-toolchain-latest
```
3. 下载源码
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
index 2dc0ae8..22a8599 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
@@ -2064,6 +2064,10 @@ public class Load {
jobInfo.add(status.getTrackingUrl());
// job detail(not used for hadoop load, just return an empty
string)
jobInfo.add("");
+ // transaction id
+ jobInfo.add(loadJob.getTransactionId());
+ // error tablets(not used for hadoop load, just return an
empty string)
+ jobInfo.add("");
loadJobInfos.add(jobInfo);
} // end for loadJobs
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
