This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris-website.git
commit c5b971179f8909ede0943a5deb160b84aa40c7ed Author: morningman <morning...@163.com> AuthorDate: Sun Jan 2 21:50:48 2022 +0800 [blog][weekly] Add odbc tutorials and weekly 1. Add odbc tutorials of ubuntu + mysql/pg/oracle 2. Add weekly 20211213-20220102 --- blogs/en/odbc-tutorial-ubuntu-mysql.md | 37 +++ blogs/en/odbc-tutorial-ubuntu-oracle.md | 36 ++ blogs/en/odbc-tutorial-ubuntu-pg.md | 37 +++ blogs/en/weekly-20220102.md | 144 ++++++++ blogs/zh-CN/odbc-tutorial-ubuntu-mysql.md | 367 +++++++++++++++++++++ blogs/zh-CN/odbc-tutorial-ubuntu-oracle.md | 506 +++++++++++++++++++++++++++++ blogs/zh-CN/odbc-tutorial-ubuntu-pg.md | 411 +++++++++++++++++++++++ blogs/zh-CN/weekly-20220102.md | 144 ++++++++ 8 files changed, 1682 insertions(+) diff --git a/blogs/en/odbc-tutorial-ubuntu-mysql.md b/blogs/en/odbc-tutorial-ubuntu-mysql.md new file mode 100644 index 0000000..5619a0b --- /dev/null +++ b/blogs/en/odbc-tutorial-ubuntu-mysql.md @@ -0,0 +1,37 @@ +--- +{ + "title": "ODBC External Tables Tutorial(1): MySQL + Ubuntu", + "description": "Details on how to connect to MySQL databases using ODBC external tables on Ubuntu", + "date": "2022-01-01", + "metaTitle": "ODBC External Tables Tutorial(1)", + "isArticle": true, + "language": "en", + "author": "张家锋", + "layout": "Article", + "sidebar": false +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +(TODO) + +This artical only has Chinese version. If you are willing to translate it into English, please contact us by sending email to d...@doris.apache.org. We are looking forward to your contribution! + diff --git a/blogs/en/odbc-tutorial-ubuntu-oracle.md b/blogs/en/odbc-tutorial-ubuntu-oracle.md new file mode 100644 index 0000000..375d0f2 --- /dev/null +++ b/blogs/en/odbc-tutorial-ubuntu-oracle.md @@ -0,0 +1,36 @@ +--- +{ + "title": "ODBC External Tables Tutorial(3): Oracle + Ubuntu", + "description": "Details on how to connect to Oracle databases using ODBC external tables on Ubuntu", + "date": "2022-01-01", + "metaTitle": "ODBC External Tables Tutorial(3)", + "isArticle": true, + "language": "en", + "author": "张家锋", + "layout": "Article", + "sidebar": false +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +(TODO) + +This artical only has Chinese version. If you are willing to translate it into English, please contact us by sending email to d...@doris.apache.org. We are looking forward to your contribution! \ No newline at end of file diff --git a/blogs/en/odbc-tutorial-ubuntu-pg.md b/blogs/en/odbc-tutorial-ubuntu-pg.md new file mode 100644 index 0000000..1154955 --- /dev/null +++ b/blogs/en/odbc-tutorial-ubuntu-pg.md @@ -0,0 +1,37 @@ +--- +{ + "title": "ODBC External Tables Tutorial(2): PostgreSQL + Ubuntu", + "description": "Details on how to connect to PostgreSQL databases using ODBC external tables on Ubuntu", + "date": "2022-01-01", + "metaTitle": "ODBC External Tables Tutorial(2)", + "isArticle": true, + "language": "en", + "author": "张家锋", + "layout": "Article", + "sidebar": false +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +(TODO) + +This artical only has Chinese version. If you are willing to translate it into English, please contact us by sending email to d...@doris.apache.org. We are looking forward to your contribution! + diff --git a/blogs/en/weekly-20220102.md b/blogs/en/weekly-20220102.md new file mode 100644 index 0000000..24ba69d --- /dev/null +++ b/blogs/en/weekly-20220102.md @@ -0,0 +1,144 @@ +--- +{ + "title": "[Doris Weekly] 20211213-20220102", + "description": "Doris Weekly", + "date": "2022-01-02", + "metaTitle": "[Doris Weekly] 20211213-20220102", + "isArticle": true, + "language": "en", + "author": "morningman", + "layout": "Article", + "sidebar": false +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +## Statistics + +A total of 37 authors have submitted 82 Commits. Thanks to the following authors for their contributions (Github IDs, in alphabetical order). + +BiteTheDDDDt, CalvinKirs, EmmyMiao87, GoGoWen, HappenLee, Henry2SS, JNSimba, aihai, caiconghui, dh-cloud, gaodayue, hf200012, jackwener, liutang123, luozenglin, luzhijing, morningman, pengxiangyu, qidaye, qzsee, spaces-X, tianhui5, tinkerrrr, wangshuo128, wayofeng, weizuo93, whutpencil, xiedeyantu, xinyiZzz, xtr1993, xuzifu666, yangzhg, yjant, zbtzbtzbt, zenoyang, zhangstar333, zhengshiJ + +In the last 3 weeks, 15882 new lines of code were added and 6966 lines of code were deleted. + +## Main progress + +### New features + +* https://github.com/apache/incubator-doris/pull/7276 + + Support return binary data of HLL/BITMAP column directly for external parsing. + +* https://github.com/apache/incubator-doris/pull/7255 + + Support for Lateral View column to row transitions. It is convenient to expand bitmap, stirng or json array data into multiple rows for subsequent processing. + +* https://github.com/apache/incubator-doris/pull/7379 + + `bitmap_union` and `bitmap_intersect` support multiple bitmap arguments. This is to improve the efficiency of intersecting and merging multiple bitmaps. + +* https://github.com/apache/incubator-doris/pull/7464 + + Support SM3/SM4 national secret algorithm functions. + +### Work-in-Progess + +* https://github.com/apache/incubator-doris/pull/7098 + + Support cold data dumping to object storage (under development). + +### Bug Fixes + +* https://github.com/apache/incubator-doris/pull/7383 + + Fix Segment Cache not handling expired cache correctly in some cases. + +* https://github.com/apache/incubator-doris/pull/7373 + + Fix several bugs in BE side constant folding code. + +* https://github.com/apache/incubator-doris/pull/7371 + + Fix a problem with Broker Load scheduling logic that could cause tasks to not be scheduled in some cases. + +* https://github.com/apache/incubator-doris/pull/7375 + + Fix an issue where some logical view queries cause SQL Cache not to be updated. + +* https://github.com/apache/incubator-doris/pull/7407 + + Optimize the logic of SQL block rule so that it only works for select statements. + +* https://github.com/apache/incubator-doris/pull/7421 + + Fix an incorrect result of round() function. + +* https://github.com/apache/incubator-doris/pull/7362 + + Fix a logical error when rewriting the count function for materialized views in some cases. + +* https://github.com/apache/incubator-doris/pull/7495 + + Fix an alias parsing error in some queries under case-insensitive table name setting. + +* https://github.com/apache/incubator-doris/pull/7440 + + Fix a bug that bloom filter handles empty strings incorrectly. + +* https://github.com/apache/incubator-doris/pull/7411 + + Fix an issue where Grouping Set function does not handle expressions correctly in some cases. + +### Feature improvements + +* https://github.com/apache/incubator-doris/pull/7232 + + Flink-Connector supports reading Doris data concurrently. + +* https://github.com/apache/incubator-doris/pull/7348 + + Optimize the efficiency of reading Unique Key tables at Reader level. + +* https://github.com/apache/incubator-doris/pull/7454 + + Optimize the memory overhead of BE when importing large amounts of non-compliant data. + +* https://github.com/apache/incubator-doris/pull/7096 + + Query Optimizer supports more predicate passing optimizations. + +* https://github.com/apache/incubator-doris/pull/7435 + + Optimize Flink-Connector's BE selection logic to select only surviving BE nodes. + +* https://github.com/apache/incubator-doris/pull/7493 + + Fix some default parameters of brpc to reduce the probability of BRPC network transmission errors. + +* https://github.com/apache/incubator-doris/pull/7434 + + Refactor the query optimizer partition cropping logic to support partition cropping capabilities for more predicate conditions. + +### Other + +* https://github.com/apache/incubator-doris/pull/7536 + + Upgrading Log4j to 2.7.1 has fixed security vulnerability CVE-2021-44832 diff --git a/blogs/zh-CN/odbc-tutorial-ubuntu-mysql.md b/blogs/zh-CN/odbc-tutorial-ubuntu-mysql.md new file mode 100644 index 0000000..b3f285b --- /dev/null +++ b/blogs/zh-CN/odbc-tutorial-ubuntu-mysql.md @@ -0,0 +1,367 @@ +--- +{ + "title": "ODBC 外表使用教程(一):MySQL + Ubuntu", + "description": "详细介绍如何在 Ubuntu 系统下,使用 ODBC 外表功能连接 MySQL 数据库", + "date": "2022-01-01", + "metaTitle": "ODBC 外表使用教程(一)", + "isArticle": true, + "language": "zh-CN", + "author": "张家锋", + "layout": "Article", + "sidebar": false +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +之前写了在Centos下Apache Doris 外表使用方法及注意实现,但是不少用户是使用ubuntu系统,在这个系统下很多用户遇到ODBC外表导致BE服务宕机的问题,对此我专门在ubuntu 18.04下进行了测试。 + +[[toc]] + +## 1.软件环境 + +1. 操作系统:ubuntu 18.04 +2. Apache Doris :0.15 +3. mysql 数据库:8.0.27-0ubuntu0.20.04.1 for Linux on x86_64 +4. UnixODBC:2.3.4 +5. Mysql Connector ODBC :5.3.13、8.0.11 + +## 2.安装ODBC驱动 + +首先我们安装unixODBC驱动、这里直接给出驱动的下载地址及安装命令 + +```shell +$> sudo wget ftp://ftp.unixodbc.org/pub/unixODBC/unixODBC-2.3.4.tar.gz +$> tar -xvzf unixODBC-2.3.4.tar.gz +$> cd unixODBC-2.3.4/ +$> sudo ./configure --prefix=/usr/local/unixODBC-2.3.7 --includedir=/usr/include --libdir=/usr/lib -bindir=/usr/bin --sysconfdir=/etc +$> make +$> sudo make install +``` + +安装成功后,unixODBC所需的头文件都被安装到了/usr/inlucde下,编译好的库文件安装到了/usr/lib下,与unixODBC相关的可执行文件安装到了/usr/bin下,配置文件放到了/etc下。 + +验证安装是否成功 + +``` +$> odbcinst -j +unixODBC 2.3.4 +DRIVERS............: /etc/odbcinst.ini +SYSTEM DATA SOURCES: /etc/odbc.ini +FILE DATA SOURCES..: /etc/ODBCDataSources +USER DATA SOURCES..: /root/.odbc.ini +SQLULEN Size.......: 8 +SQLLEN Size........: 8 +SQLSETPOSIROW Size.: 8 +``` + +## 3.安装Mysql ODBC驱动 + +**这里我默认你是知道Mysql的安装方法,或者你已经有了Mysql数据库**,对Mysql的安装配置就不在讲了,如果这块不清楚,请自行搜索。 + +从mysql 站点下载对应的驱动 + +https://dev.mysql.com/downloads/connector/odbc/ + +这里我们可以使用 8.0.11 和 5.3.13 版本。本文使用 8.0.11 版本进行说明。(8.0.26 版本有兼容性问题,请勿使用。后面有说明) + +`mysql-connector-odbc-8.0.11-linux-glibc2.12-x86-64bit.tar.gz` + +下载后解压,我这里为了测试方便,直接将解压后的目录重命名使用了,没有将这个目录下的bin和lib拷贝到/usr/local/目录下,具体的操作命令: + +```shell +tar zxvf mysql-connector-odbc-8.0.11-linux-glibc2.12-x86-64bit.tar.gz +mv mysql-connector-odbc-8.0.11-linux-glibc2.12-x86-64bit mysql-odbc-8.0.11 +``` + +注册Mysql驱动 + +``` +myodbc-installer -a -d -n "MySQL ODBC 8.0.11 Unicode Driver" -t "Driver=/root/mysql-odbc-8.0.11/lib/libmyodbc8w.so" +myodbc-installer -a -d -n "MySQL ODBC 8.0.11 ANSI Driver" -t "Driver=/root/mysql-odbc-8.0.11/lib/libmyodbc8a.so" +``` + +然后查看是否注册成功 + +使用这个命令:myodbc-installer -d -l + +```myodbc-installer -d -l +MySQL ODBC 5.0 Unicode Driver +MySQL ODBC 5.0 ANSI Driver +MySQL ODBC 8.0 Unicode Driver +MySQL ODBC 8.0 +MySQL ODBC 8.0.11 Unicode Driver +MySQL ODBC 8.0.11 ANSI Driver +``` + +这里我安装了上面说的三个版本的驱动,另外两个版本的驱动安装方式一样 + +## 4.验证通过ODBC访问Mysql + +我们去配置ODBC访问Mysql的参数 + +编辑/etc/odbc.ini文件,加入下面的内容,将信息替换成你自己的 + +``` +[mysql] +Description = Data source MySQL +Driver = MySQL ODBC 8.0 Unicode Driver +Server = localhost +Host = localhost +Database = demo +Port = 3306 +User = root +Password = zhangfeng +``` + +然后我们通过: + +isql -v mysql + +```sql +isql -v mysql ++---------------------------------------+ +| Connected! | +| | +| sql-statement | +| help [tablename] | +| quit | +| | ++---------------------------------------+ +``` + +说明我们ODBC配置成功。 + +## 5.测试Apache Doris ODBC外表 + +Doris的安装配置参考我的博客:[Apache Doris安装部署](https://mp.weixin.qq.com/s/pnZgya-DcjhaktedIQcBOA),或者官网的文档。 + +首先我们在Mysql数据库见了一个demo库及相应的表 + +```sql +CREATE TABLE `test_cdc` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=91234 DEFAULT CHARSET=utf8mb4; +``` + +对应的测试数据 + +```sql +INSERT INTO `test_cdc` VALUES (123, 'this is a update'); +INSERT INTO `test_cdc` VALUES (1212, '测试flink CDC'); +INSERT INTO `test_cdc` VALUES (1234, '这是测试'); +INSERT INTO `test_cdc` VALUES (11233, 'zhangfeng_1'); +INSERT INTO `test_cdc` VALUES (21233, 'zhangfeng_2'); +INSERT INTO `test_cdc` VALUES (31233, 'zhangfeng_3'); +INSERT INTO `test_cdc` VALUES (41233, 'zhangfeng_4'); +INSERT INTO `test_cdc` VALUES (51233, 'zhangfeng_5'); +INSERT INTO `test_cdc` VALUES (61233, 'zhangfeng_6'); +INSERT INTO `test_cdc` VALUES (71233, 'zhangfeng_7'); +INSERT INTO `test_cdc` VALUES (81233, 'zhangfeng_8'); +INSERT INTO `test_cdc` VALUES (91233, 'zhangfeng_9'); +``` + +下面建Doris的ODBC外表,这里我们是通过ODBC_Resource来创建ODBC外表,这也是推荐的方式,这样ODBC resource是可以复用 + +首先我们在BE节点的conf/odbcinst.ini,添加我们的刚才注册的8.0.21的ODBC驱动([MySQL ODBC 8.0.21])。 + +注意这里我们使用 `libmyodbc8w.so` 而不是 `libmyodbc8a.so`。因为 Doris 目前仅支持 Unicode Driver,不支持 ANSI Driver。 + +``` +[MySQL ODBC 8.0.11] +Description = ODBC for MySQL +Driver=/root/mysql-odbc-8.0.11/lib/libmyodbc8w.so +FileUsage = 1 +``` + +首先我们创建resource + +```sql + CREATE EXTERNAL RESOURCE `mysql_8_0_11` + PROPERTIES ( +"host" = "localhost", + "port" = "3306", + "user" = "root", + "password" = "zhangfeng", + "database" = "demo", + "table" = "test_cdc", + "driver" = "MySQL ODBC 8.0.11", --注意这里的名称要和我们上面红框标识部分的[]里的名称一致 + "odbc_type" = "mysql", + "type" = "odbc_catalog" + ); +``` + +基于这个resource创建ODBC外表 + +```、sql +CREATE EXTERNAL TABLE `test_odbc_8_0_11` ( + `id` int NOT NULL , + `name` varchar(255) null +) ENGINE=ODBC +COMMENT "ODBC" +PROPERTIES ( +"odbc_catalog_resource" = "mysql_8_0_11", --这里的名称就是我们上面定义的resource的名称 +"database" = "demo", +"table" = "test_cdc" +); +``` + +执行结果 + +```sql +mysql> use demo; +Reading table information for completion of table and column names +You can turn off this feature to get a quicker startup with -A + +Database changed + +mysql> CREATE EXTERNAL RESOURCE `mysql_8_0_11` + -> PROPERTIES ( + -> "host" = "localhost", + -> "port" = "3306", + -> "user" = "root", + -> "password" = "zhangfeng", + -> "database" = "demo", + -> "table" = "test_cdc", + -> "driver" = "MySQL ODBC 8.0.11", + -> "odbc_type" = "mysql", + -> "type" = "odbc_catalog" + -> ); +Query OK, 0 rows affected (0.01 sec) + +mysql> CREATE EXTERNAL TABLE `test_odbc_8_0_11` ( + -> `id` int NOT NULL , + -> `name` varchar(255) null + -> ) ENGINE=ODBC + -> COMMENT "ODBC" + -> PROPERTIES ( + -> "odbc_catalog_resource" = "mysql_8_0_11", + -> "database" = "demo", + -> "table" = "test_cdc" + -> ); +Query OK, 0 rows affected (0.01 sec) +``` + +一切显示正常,下面是见证奇迹的时候,我们执行查询外表操作 + +```sql +select * from test_odbc_8_0_11; ++-------+------------------+ +| id | name | ++-------+------------------+ +| 123 | this is a update | +| 1212 | 测试flink CDC | +| 1234 | 这是测试 | +| 11233 | zhangfeng_1 | +| 21233 | zhangfeng_2 | +| 31233 | zhangfeng_3 | +| 41233 | zhangfeng_4 | +| 51233 | zhangfeng_5 | +| 61233 | zhangfeng_6 | +| 71233 | zhangfeng_7 | +| 81233 | zhangfeng_8 | +| 91233 | zhangfeng_9 | ++-------+------------------+ +12 rows in set (0.01 sec) +``` + +可以看到,查询正常。至此,完成安装。 + +同样,我们也可以使用 5.3.13 进行测试。同样可以正常工作。 + +## 6.不兼容的 Driver 版本 + +如果我们使用了不兼容的 MySQL Driver,如 8.0.26,则当查询 ODBC 外表时,可能出现以下错误: + +```sql +select * from test_odbc_8_0_26; +ERROR 1064 (HY000): errCode = 2, detailMessage = there is no scanNode Backend. [10002: in black list(io.grpc.StatusRuntimeException: UNAVAILABLE: Network closed for unknown reason)] +``` + +这个时候显示BE节点挂了,我们通过show backends命令去查看也是显示BE节点挂了 + +``` +mysql> show backends\G; +*************************** 1. row *************************** + BackendId: 10002 + Cluster: default_cluster + IP: 172.16.192.81 + HeartbeatPort: 9050 + BePort: 9060 + HttpPort: 8040 + BrpcPort: 8060 + LastStartTime: 2021-12-31 10:43:16 + LastHeartbeat: 2021-12-31 11:03:00 + Alive: false + SystemDecommissioned: false +ClusterDecommissioned: false + TabletNum: 0 + DataUsedCapacity: 0.000 + AvailCapacity: 83.224 GB + TotalCapacity: 98.305 GB + UsedPct: 15.34 % + MaxDiskUsedPct: 15.34 % + Tag: {"location" : "default"} + ErrMsg: + Version: 0.15.1-rc09-Unknown + Status: {"lastSuccessReportTabletsTime":"2021-12-31 11:02:22","lastStreamLoadTime":-1} +1 row in set (0.00 sec) +``` + +查看BE的日志(be.out)会发现有如下类似错误堆栈: + +``` +*** Aborted at 1640918068 (unix time) try "date -d @1640918068" if you are using GNU date *** +PC: @ 0x7f8caaf29b7e (unknown) +*** SIGSEGV (@0x0) received by PID 56420 (TID 0x7f8c62370700) from PID 0; stack trace: *** + @ 0x3022682 google::(anonymous namespace)::FailureSignalHandler() + @ 0x7f8cab0f93c0 (unknown) + @ 0x7f8caaf29b7e (unknown) + @ 0x34ef4ac getrn + @ 0x34ef722 lh_insert + @ 0x34abd14 OBJ_NAME_add + @ 0x7f8c09f9d115 ossl_init_ssl_base_ossl_ + @ 0x7f8cab0f647f __pthread_once_slow + @ 0x7f8c0a3af194 CRYPTO_THREAD_run_once + @ 0x7f8c09f9cf87 OPENSSL_init_ssl + @ 0x7f8c0a77e504 ssl_start() + @ 0x7f8c0a751f55 mysql_server_init + @ 0x7f8c0a75c425 mysql_init + @ 0x7f8c0a72cbcd DBC::connect() + @ 0x7f8c0a72f403 MySQLDriverConnect() + @ 0x7f8c0a74ea08 SQLDriverConnectW + @ 0x3b060ec SQLDriverConnect + @ 0x1ec046b doris::ODBCConnector::open() + @ 0x1eb9706 doris::OdbcScanNode::open() + @ 0x189e6e9 doris::PlanFragmentExecutor::open_internal() + @ 0x189fb4c doris::PlanFragmentExecutor::open() + @ 0x181b70e doris::FragmentExecState::execute() + @ 0x181f706 doris::FragmentMgr::_exec_actual() + @ 0x1828d4f std::_Function_handler<>::_M_invoke() + @ 0x198a963 doris::ThreadPool::dispatch_thread() + @ 0x1984aac doris::Thread::supervise_thread() + @ 0x7f8cab0ed609 start_thread + @ 0x7f8caaec5293 clone + @ 0x0 (unknown) +``` + +这是因为驱动版本不兼容导致。所以在 Ubuntu 环境下,建议使用 MySQL Driver 8.0.11 或 5.3.13 这两个版本。 \ No newline at end of file diff --git a/blogs/zh-CN/odbc-tutorial-ubuntu-oracle.md b/blogs/zh-CN/odbc-tutorial-ubuntu-oracle.md new file mode 100644 index 0000000..216b255 --- /dev/null +++ b/blogs/zh-CN/odbc-tutorial-ubuntu-oracle.md @@ -0,0 +1,506 @@ +--- +{ + "title": "ODBC 外表使用教程(三):Oracle + Ubuntu", + "description": "详细介绍如果在 Ubuntu 系统下,使用 ODBC 外表功能连接 Oracle 数据库", + "date": "2022-01-01", + "metaTitle": "ODBC 外表使用教程(三)", + "isArticle": true, + "language": "zh-CN", + "author": "张家锋", + "layout": "Article", + "sidebar": false +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +本文档详细介绍如何在 Ubuntu 环境下,使用 Doris 的 ODBC 外表功能连接 Oracle 数据库。 + +[[toc]] + +## 1.软件环境 + +1. 操作系统:ubuntu 18.04 +2. Apache Doris :0.15 +3. Postgresql数据库:oracle 19c +4. UnixODBC:2.3.4 +5. Oracle ODBC :instantclient-odbc-linux.x64-19.13.0.0.0dbru + +## 2.Oracle安装部署 + +因为是测试没有Oracle的环境,由于安装比较繁琐耗时,所以通过Docker进行快速安装部署。 + +**本章节仅记录笔者安装 Oracle 的过程。如果你已安装,可以跳到第3节继续阅读。** + +### 2.1 制作镜像 + +安装方法是通过官方dockerfile自己编译镜像安装, + +```shell +$> git clone https://github.com/oracle/docker-images.git +Cloning into 'docker-images'... +remote: Enumerating objects: 77, done. +remote: Counting objects: 100% (77/77), done. +remote: Compressing objects: 100% (52/52), done. +remote: Total 9878 (delta 25), reused 55 (delta 23), pack-reused 9801 +Receiving objects: 100% (9878/9878), 10.20 MiB | 2.47 MiB/s, done. +Resolving deltas: 100% (5686/5686), done. +$> cd docker-images/OracleDatabase/SingleInstance/dockerfiles/ +$> ls -al +total 44 +drwxr-xr-x 9 root root 4096 Jan 1 11:51 ./ +drwxr-xr-x 7 root root 4096 Jan 1 11:51 ../ +drwxr-xr-x 2 root root 4096 Jan 1 11:51 11.2.0.2/ +drwxr-xr-x 2 root root 4096 Jan 1 11:51 12.1.0.2/ +drwxr-xr-x 2 root root 4096 Jan 1 11:51 12.2.0.1/ +drwxr-xr-x 2 root root 4096 Jan 1 11:51 18.3.0/ +drwxr-xr-x 2 root root 4096 Jan 1 11:51 18.4.0/ +drwxr-xr-x 2 root root 4096 Jan 1 12:04 19.3.0/ +drwxr-xr-x 2 root root 4096 Jan 1 11:51 21.3.0/ +-rwxr-xr-x 1 root root 7091 Jan 1 11:51 buildContainerImage.sh* ---制作镜像的命令 +``` + +目前支持以上几个版本的docker安装。 + +我们这里要制作安装的是19.3.0这个版本 + +```shell +$> cd 19.3.0/ && ls -al +total 2988092 +drwxr-xr-x 2 root root 4096 Jan 1 12:04 ./ +drwxr-xr-x 9 root root 4096 Jan 1 11:51 ../ +-rwxr-xr-x 1 root root 1044 Jan 1 11:51 checkDBStatus.sh* +-rwxr-xr-x 1 root root 904 Jan 1 11:51 checkSpace.sh* +-rw-r--r-- 1 root root 63 Jan 1 11:51 Checksum.ee +-rw-r--r-- 1 root root 63 Jan 1 11:51 Checksum.se2 +-rwxr-xr-x 1 root root 7634 Jan 1 11:51 createDB.sh* +-rw-r--r-- 1 root root 9204 Jan 1 11:51 dbca.rsp.tmpl +-rw-r--r-- 1 root root 6878 Jan 1 11:51 db_inst.rsp +-rw-r--r-- 1 root root 4398 Jan 1 11:51 Dockerfile +-rwxr-xr-x 1 root root 2712 Jan 1 11:51 installDBBinaries.sh* +-rw-r--r-- 1 root root 3059705302 Apr 24 2019 LINUX.X64_193000_db_home.zip ---oracle安装介质文件 +-rw-r--r-- 1 root root 2008 Jan 1 11:51 relinkOracleBinary.sh +-rwxr-xr-x 1 root root 7743 Jan 1 11:51 runOracle.sh* +-rwxr-xr-x 1 root root 1021 Jan 1 11:51 runUserScripts.sh* +-rwxr-xr-x 1 root root 795 Jan 1 11:51 setPassword.sh* +-rwxr-xr-x 1 root root 1057 Jan 1 11:51 setupLinuxEnv.sh* +-rwxr-xr-x 1 root root 679 Jan 1 11:51 startDB.sh* +$> cat Dockerfile|grep INSTALL_FILE_1 + INSTALL_FILE_1="LINUX.X64_193000_db_home.zip" \ ---这里我们可以看到Oracle的安装介质文件名 +COPY --chown=oracle:dba $INSTALL_FILE_1 $INSTALL_RSP $INSTALL_DB_BINARIES_FILE $INSTALL_DIR/ +``` + +我们需要下载Oracle的安装介质文件 + +下载地址:https://www.oracle.com/database/technologies/oracle19c-linux-downloads.html#license-lightbox + +这里需要登录,下载完成之后将zip包放到这个目录下,然后执行下面命令: + +``` +$> ./buildDockerImage.sh -v 19.3.0 -e +``` + +`e表示企业版` + +``` +Successfully built 701b50f5832a +Successfully tagged oracle/database:19.3.0-ee + + + Oracle Database Docker Image for 'ee' version 19.3.0 is ready to be extended: + + --> oracle/database:19.3.0-ee + + Build completed in 2624 seconds. +``` + +当出现上面的提示,就表示镜像制作完成了,剩下就是利用该镜像启动容器即可。 + +### 2.2 启动容器,安装数据库 + +注意:oracle 企业版的docker run的命令格式如下(XE版的都有所区别): + +```sql +$> docker run --name oracle-19c \ +-p 1521:1521 -p 5500:5500 \ +-e ORACLE_SID=lei \ ---这里是你安装数据的时候创建的数据库SID +-e ORACLE_PDB=leipdb \ +-e ORACLE_PWD=Oracle \ +-v /oracle/oradata:/opt/oracle/oradata \ +oracle/database:19.3.0-ee +``` + +> 注:`/oracle/oradata`目录权限一定要正确,在容器中oracle用户的uid是54321,所以要保证容器内的oracle用户有权限读写该目录。 + +> 注: 如果"DATABASE IS READY TO USE!"提示信息,表示数据库已成功安装了。 + +### 2.3 验证Oracle安装 + +登陆主机或数据库进行操作 + +``` +$> docker exec -it oracle-19c /bin/bash +$> export ORACLE_SID=LEI +$> sqlplus / as sysdba + +SQL*Plus: Release 19.0.0.0.0 - Production on Sun Jan 2 02:28:55 2022 +Version 19.3.0.0.0 + +Copyright (c) 1982, 2019, Oracle. All rights reserved. + + +Connected to: +Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production +Version 19.3.0.0.0 + +SQL> show pdbs; + + CON_ID CON_NAME OPEN MODE RESTRICTED +---------- ------------------------------ ---------- ---------- + 2 PDB$SEED READ ONLY NO + 3 LEIPDB READ WRITE NO +SQL> +``` + +至此就完成了docker安装Oracle 19c! + +然后我们就可以创建数据库、创建表、导入数据进行测试 + +我们首先创建一个表空间,然后创建用户挂在到这个表空间下 + +```sql +---创建表空间 +create tablespace demo1 datafile '/opt/oracle/oradata/demo1.dbf' size 200M; +---创建用户并给用户设置默认的表空间 +create user C##dbuser identified by "zhangfeng" default tablespace demo1; +---给用户授权 +grant connect,resource,dba to C##dbuser; +grant create session to C##dbuser; +``` + +然后我们可以使用Navicat或者SQLPlus等其他客户端使用刚才创建的用户连接Oracle,去创建表,导入数据 + +下面是我测试的建表语句及插入的示例数据 + +```sql +CREATE TABLE persons( + person_id NUMBER GENERATED BY DEFAULT AS IDENTITY, + first_name VARCHAR2(50) NOT NULL, + last_name VARCHAR2(50) NOT NULL, + PRIMARY KEY(person_id) +); + +INSERT INTO "C##DBUSER"."PERSONS" VALUES ('1', 'zhang', 'feng'); +INSERT INTO "C##DBUSER"."PERSONS" VALUES ('2', '张峰', '峰'); +INSERT INTO "C##DBUSER"."PERSONS" VALUES ('3', 'zhangfeng', '峰值'); +``` + +## 3.安装unixODBC驱动 + +首先我们安装unixODBC驱动、这里直接给出驱动的下载地址及安装命令 + +```shell +$>sudo wget ftp://ftp.unixodbc.org/pub/unixODBC/unixODBC-2.3.4.tar.gz +$>tar -xvzf unixODBC-2.3.4.tar.gz +$>cd unixODBC-2.3.4/ +$>sudo ./configure --prefix=/usr/local/unixODBC-2.3.7 --includedir=/usr/include --libdir=/usr/lib -bindir=/usr/bin --sysconfdir=/etc +$>make +$>sudo make install +``` + +安装成功后,unixODBC所需的头文件都被安装到了/usr/inlucde下,编译好的库文件安装到了/usr/lib下,与unixODBC相关的可执行文件安装到了/usr/bin下,配置文件放到了/etc下。 + +验证安装是否成功 + +``` +$> odbcinst -j +unixODBC 2.3.4 +DRIVERS............: /etc/odbcinst.ini +SYSTEM DATA SOURCES: /etc/odbc.ini +FILE DATA SOURCES..: /etc/ODBCDataSources +USER DATA SOURCES..: /root/.odbc.ini +SQLULEN Size.......: 8 +SQLLEN Size........: 8 +SQLSETPOSIROW Size.: 8 +``` + +## 4.安装Oracle ODBC驱动及测试 + +### 4.1 安装驱动 + +我们需要下载下面这几个安装包: + +``` +oracle-instantclient19.13-basic-19.13.0.0.0-2.x86_64.rpm +oracle-instantclient19.13-devel-19.13.0.0.0-2.x86_64.rpm +oracle-instantclient19.13-odbc-19.13.0.0.0-2.x86_64.rpm +oracle-instantclient19.13-sqlplus-19.13.0.0.0-2.x86_64.rpm +``` + +下面是下载地址: + +``` +https://download.oracle.com/otn_software/linux/instantclient/1913000/oracle-instantclient19.13-sqlplus-19.13.0.0.0-2.x86_64.rpm + +https://download.oracle.com/otn_software/linux/instantclient/1913000/oracle-instantclient19.13-devel-19.13.0.0.0-2.x86_64.rpm + +https://download.oracle.com/otn_software/linux/instantclient/1913000/oracle-instantclient19.13-odbc-19.13.0.0.0-2.x86_64.rpm + +https://download.oracle.com/otn_software/linux/instantclient/1913000/oracle-instantclient19.13-basic-19.13.0.0.0-2.x86_64.rpm +``` + +为了在ubuntu下可以进行安装rpm包,我们还需要安装一个alien,这是一个可以将rpm包转换成deb安装包的工具 + +``` +$> sudo apt-get install alien +``` + +然后执行安装上面四个包 + +``` +$> sudo alien -i oracle-instantclient19.13-basic-19.13.0.0.0-2.x86_64.rpm +$> sudo alien -i oracle-instantclient19.13-devel-19.13.0.0.0-2.x86_64.rpm +$> sudo alien -i oracle-instantclient19.13-odbc-19.13.0.0.0-2.x86_64.rpm +$> sudo alien -i oracle-instantclient19.13-sqlplus-19.13.0.0.0-2.x86_64.rpm +``` + +验证我们安装的ODBC驱动动态链接库是否正确 + +```shell +$> ldd /usr/lib/oracle/19.13/client64/lib/libsqora.so.19.1 + linux-vdso.so.1 (0x00007ffefef27000) + libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f43e80b0000) + libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f43e7f61000) + libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f43e7f3e000) + libnsl.so.1 => /lib/x86_64-linux-gnu/libnsl.so.1 (0x00007f43e7f21000) + librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f43e7f16000) + libaio.so.1 => /lib/x86_64-linux-gnu/libaio.so.1 (0x00007f43e7f11000) + libresolv.so.2 => /lib/x86_64-linux-gnu/libresolv.so.2 (0x00007f43e7ef3000) + libclntsh.so.19.1 => /usr/lib/oracle/19.13/client64/lib/libclntsh.so.19.1 (0x00007f43e3d6f000) + libclntshcore.so.19.1 => /usr/lib/oracle/19.13/client64/lib/libclntshcore.so.19.1 (0x00007f43e37cb000) + libodbcinst.so.2 => /usr/local/lib/libodbcinst.so.2 (0x00007f43e37b3000) + libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f43e35c1000) + /lib64/ld-linux-x86-64.so.2 (0x00007f43e8379000) + libnnz19.so => /usr/lib/oracle/19.13/client64/lib/libnnz19.so (0x00007f43e2f4a000) + libltdl.so.7 => /lib/x86_64-linux-gnu/libltdl.so.7 (0x00007f43e2f3d000) +``` + +### 4.2 配置环境变量 + +``` +$> sudo vi ~/.bashrc +``` + +加入下面的内容 + +``` +export ORACLE_HOME=/usr/lib/oracle/19.13/client64 +export TNS_ADMIN=$ORACLE_HOME/network/admin +export LD_LIBRARY_PATH=$ORACLE_HOME/lib +export ORACLE_SID=LEI +export PATH=$ORACLE_HOME/bin:$PATH +``` + +### 4.3 配置 tnsnames.ora 文件 + +``` +$> cd /usr/lib/oracle/19.13/client64 +$> mkdir -p network/admin +$> vi tnsnames.ora +``` + +加入下面的内容(注意修改成自己的,这里是示例) + +``` +demo = + (DESCRIPTION = + (ADDRESS_LIST = + (ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.192.81)(PORT = 1521)) + ) + (CONNECT_DATA = + ( SERVICE_NAME = LEI) + ) + ) +``` + +> 注1. 将HOST、PORT换成你自己的 +> +> 注2. SERVICE_NAME :这个是我们启动Oracle docker的时候设置的ORACLE_SID + +### 4.4 配置odbcinst.ini + +这里使用RPM包安装的额ODBC驱动动态链接库在 `/usr/lib/oracle/19.13/client64/lib/`目录下 + +``` +$> ls -al /usr/lib/oracle/19.13/client64/lib/ +total 236232 +drwxr-xr-x 3 root root 4096 Jan 2 11:04 ./ +drwxr-xr-x 5 root root 4096 Jan 1 19:47 ../ +-rw-r--r-- 1 root root 342 Nov 27 02:41 glogin.sql +lrwxrwxrwx 1 root root 21 Jan 1 19:43 libclntshcore.so -> libclntshcore.so.19.1* +-rwxr-xr-x 1 root root 8057664 Nov 27 02:39 libclntshcore.so.19.1* +lrwxrwxrwx 1 root root 17 Jan 1 19:43 libclntsh.so -> libclntsh.so.19.1* +lrwxrwxrwx 1 root root 17 Jan 1 19:43 libclntsh.so.10.1 -> libclntsh.so.19.1* +lrwxrwxrwx 1 root root 17 Jan 1 19:43 libclntsh.so.11.1 -> libclntsh.so.19.1* +lrwxrwxrwx 1 root root 17 Jan 1 19:43 libclntsh.so.12.1 -> libclntsh.so.19.1* +lrwxrwxrwx 1 root root 17 Jan 1 19:43 libclntsh.so.18.1 -> libclntsh.so.19.1* +-rwxr-xr-x 1 root root 81679160 Nov 27 02:39 libclntsh.so.19.1* +-rwxr-xr-x 1 root root 3642520 Nov 27 02:39 libipc1.so* +-rwxr-xr-x 1 root root 478728 Nov 27 02:39 libmql1.so* +-rwxr-xr-x 1 root root 5831752 Nov 27 02:39 libnnz19.so* +-rwxr-xr-x 1 root root 2342024 Nov 27 02:39 libocci.so.19.1* +-rwxr-xr-x 1 root root 130543568 Nov 27 02:39 libociei.so* +-rwxr-xr-x 1 root root 153464 Nov 27 02:39 libocijdbc19.so* +-rwxr-xr-x 1 root root 116376 Nov 27 02:39 liboramysql19.so* +-rwxr-xr-x 1 root root 1660776 Nov 27 02:41 libsqlplusic.so* +-rwxr-xr-x 1 root root 1572432 Nov 27 02:41 libsqlplus.so* +-rwxr-xr-x 1 root root 1070192 Nov 27 02:41 libsqora.so.19.1* ---这里是要使用的ODBC动态链接库 +drwxr-xr-x 3 root root 4096 Jan 1 19:43 network/ +-rw-r--r-- 1 root root 4355723 Nov 27 02:39 ojdbc8.jar +-rw-r--r-- 1 root root 313026 Nov 27 02:41 ottclasses.zip +-rw-r--r-- 1 root root 37519 Nov 27 02:39 xstreams.jar +``` + +编辑 `/etc/odbcinst.ini`,在最后加上下面的内容 + +``` +[Oracle 19 ODBC driver] +Description = Oracle ODBC driver for Oracle 19 +Driver = /usr/lib/oracle/19.13/client64/lib/libsqora.so.19.1 +Setup = +FileUsage = +CPTimeout = +CPReuse = +``` + +配置odbc.ini,在最后加上下面的内容 + +``` +[oracle] +Driver = Oracle 19 ODBC driver ---这里的名称是上面odbcinst.ini里oracle部分用[]括起来的内容 +ServerName =172.16.192.81:1521/LEI --这里是你的oracle数据ip地址,端口及SID +UserID = C##dbuser --这里是我们上面创建的用户名 +Password = zhangfeng --密码 +``` + +验证ODBC + +```sql +isql oracle ++---------------------------------------+ +| Connected! | +| | +| sql-statement | +| help [tablename] | +| quit | +| | ++---------------------------------------+ +``` + +显示一切正常 + +## 5.Apache Doris Oracle外表验证 + +### 5.1 修改配置 + +修改BE节点 `conf/odbcinst.ini` 文件,加入刚才/etc/odbcinst.ini添加的一样内容,并删除原先的Oracle配置,加上你刚才安装的,如下: + +``` +[Oracle 19 ODBC driver] +Description = Oracle ODBC driver for Oracle 19 +Driver = /usr/lib/oracle/19.13/client64/lib/libsqora.so.19.1 +``` + +### 5.2 验证 + +创建oracle的ODBC Resource + +```sql + CREATE EXTERNAL RESOURCE `oracle_19` + PROPERTIES ( + "host" = "172.16.192.81", + "port" = "1521", + "user" = "C##dbuser", + "password" = "zhangfeng", + "database" = "LEI", --这里是你的数据库示例名称,也就是我们在docker启动时的ORACLE_SID + "driver" = "Oracle 19 ODBC driver", ---这里的名称一定和你在be odbcinst.ini里的oracle部分的[]里的内容一样,重要 + "odbc_type" = "oracle", + "type" = "odbc_catalog" + ); +``` + +创建ODBC外表 + +```sql +CREATE EXTERNAL TABLE `oracle_odbc` ( + person_id int, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL +) ENGINE=ODBC +COMMENT "ODBC" +PROPERTIES ( + "odbc_catalog_resource" = "oracle_19", + "database" = "LEI", + "table" = "persons" +); +``` + +下面我们看执行结果 + +````sql +mysql> CREATE EXTERNAL RESOURCE `oracle_19` + -> PROPERTIES ( + -> "host" = "172.16.192.81", + -> "port" = "1521", + -> "user" = "C##dbuser", + -> "password" = "zhangfeng", + -> "database" = "LEI", + -> "driver" = "Oracle 19 ODBC driver", + -> "odbc_type" = "oracle", + -> "type" = "odbc_catalog" + -> ); +Query OK, 0 rows affected (0.01 sec) + +mysql> +mysql> CREATE EXTERNAL TABLE `oracle_odbc` ( + -> person_id int, + -> first_name VARCHAR(50) NOT NULL, + -> last_name VARCHAR(50) NOT NULL + -> ) ENGINE=ODBC + -> COMMENT "ODBC" + -> PROPERTIES ( + -> "odbc_catalog_resource" = "oracle_19", + -> "database" = "LEI", + -> "table" = "persons" + -> ); +Query OK, 0 rows affected (0.01 sec) + +mysql> select * from oracle_odbc; ++-----------+------------+-----------+ +| person_id | first_name | last_name | ++-----------+------------+-----------+ +| 1 | zhang | feng | +| 2 | 张峰 | 峰 | +| 3 | zhangfeng | 峰值 | ++-----------+------------+-----------+ +3 rows in set (0.06 sec) +```` + +OK一切正常,正常情况下,Oracle ODBC驱动只要你的数据库版本和你的ODBC驱动版本(大版本号对上就行)一致,就基本不会出问题。 \ No newline at end of file diff --git a/blogs/zh-CN/odbc-tutorial-ubuntu-pg.md b/blogs/zh-CN/odbc-tutorial-ubuntu-pg.md new file mode 100644 index 0000000..0cb75f4 --- /dev/null +++ b/blogs/zh-CN/odbc-tutorial-ubuntu-pg.md @@ -0,0 +1,411 @@ +--- +{ + "title": "ODBC 外表使用教程(二):PostgreSQL + Ubuntu", + "description": "详细介绍如果在 Ubuntu 系统下,使用 ODBC 外表功能连接 PostgreSQL 数据库", + "date": "2022-01-01", + "metaTitle": "ODBC 外表使用教程(二)", + "isArticle": true, + "language": "zh-CN", + "author": "张家锋", + "layout": "Article", + "sidebar": false +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +本文档详细介绍如何在 Ubuntu 环境下,使用 Doris 的 ODBC 外表功能连接 PostgreSQL 数据库。 + +[[toc]] + +## 1.软件环境 + +1. 操作系统:ubuntu 18.04 +2. Apache Doris :0.15 +3. Postgresql数据库:PostgreSQL 12.9 +4. UnixODBC:2.3.4 +5. PostgreSQL ODBC :psqlodbc-12.02.0000 + +## 2.安装ODBC驱动 + +首先我们安装unixODBC驱动、这里直接给出驱动的下载地址及安装命令 + +```shell +$> sudo wget ftp://ftp.unixodbc.org/pub/unixODBC/unixODBC-2.3.4.tar.gz +$> tar -xvzf unixODBC-2.3.4.tar.gz +$> cd unixODBC-2.3.4/ +$> sudo ./configure --prefix=/usr/local/unixODBC-2.3.7 --includedir=/usr/include --libdir=/usr/lib -bindir=/usr/bin --sysconfdir=/etc +$> make +$> sudo make install +``` + +安装成功后,unixODBC所需的头文件都被安装到了/usr/inlucde下,编译好的库文件安装到了/usr/lib下,与unixODBC相关的可执行文件安装到了/usr/bin下,配置文件放到了/etc下。 + +验证安装是否成功 + +``` +$> odbcinst -j +unixODBC 2.3.4 +DRIVERS............: /etc/odbcinst.ini +SYSTEM DATA SOURCES: /etc/odbc.ini +FILE DATA SOURCES..: /etc/ODBCDataSources +USER DATA SOURCES..: /root/.odbc.ini +SQLULEN Size.......: 8 +SQLLEN Size........: 8 +SQLSETPOSIROW Size.: 8 +``` + +## 3.安装Postgresql数据库 + +**这一章节,仅记录笔者安装 Postgresql 的过程。如果你已经安装了 Postgresql,这可以直接跳到第4节继续阅读。** + +Ubuntu的默认存储库包含Postgres软件包,因此您可以使用`apt`安装这些软件包。 + +安装之前先用`apt`更新一下本地软件包,然后,安装`Postgres`包和一个附加实用程序和功能的`- managed`包: + +```shell +$> sudo apt update +$> sudo apt install postgresql postgresql-contrib +``` + +现在已经安装了该软件,我们可以了解它的工作原理以及它与您可能使用的类似数据库管理系统的不同之处。 + +### 3.1 使用PostgreSQL roles和数据库 + +默认情况下,Postgres使用称为“roles”的概念来处理身份验证和授权。在某些方面,这些类似于常规的Unix风格帐户,但Postgres不区分用户和组,而是更喜欢更灵活的术语“roles”。 + +安装后,Postgres设置为使用*ident身份*验证,这意味着它将Postgres roles与匹配的Unix / Linux系统帐户相关联。如果Postgres中存在roles,则具有相同名称的Unix / Linux用户名可以作为该roles登录。 + +安装过程创建了一个名为**postgres**的用户帐户,该帐户与默认的Postgres roles相关联。要使用Postgres,您可以登录该帐户。 + +有几种方法可以使用此帐户访问Postgres。 + +### 3.2 切换到postgres帐户 + +输入以下内容切换到服务器上的**postgres**帐户: + +```shell +$> sudo -i -u postgres +``` + +您现在可以通过输入以下内容立即访问Postgres: + +```shell +$> psql +``` + +这将使您进入PostgreSQL提示符,从此处您可以立即与数据库管理系统进行交互。 + +输入以下命令退出PostgreSQL提示符: + +```shell +postgres=# \q +``` + +### 3.3 在不切换帐户的情况下访问Postgres + +您也可以让**postgres**帐户用`sudo`运行您想要的命令。 + +例如,在最后一个示例中,您被指示通过首先切换到**postgres**用户然后运行`psql`以打开Postgres提示来进入Postgres提示。您可以通过`psql`以**postgres**用户身份运行单个命令来一步完成此操作`sudo`,如下所示: + +```shell +$> sudo -u postgres psql +``` + +这将直接登录到Postgres。 + +同样,您可以通过输入以下内容退出交互式Postgres会话: + +```shell +postgres=# \q +``` + +许多用例需要多个Postgres roles。继续阅读以了解如何配置这些 + +### 3.4 创建用户,数据库及表 + +使用默认用户登录postgresql创建用户、创建数据库及完成授权 + +```shell +$> sudo -u postgres psql +``` + +创建数据库新用户,如 dbuser: + +```sql +postgres=# CREATE USER dbuser WITH PASSWORD 'zhangfeng'; +``` + +注意: + +* 语句要以分号结尾。 +* 密码要用单引号括起来。 +* 创建用户数据库,你也可以通过你创建的用户登录进去以后创建数据库,如demo: + +```sql +postgres=# CREATE DATABASE demo OWNER dbuser; +``` + +将demo数据库的所有权限都赋予dbuser: + +```sql +postgres=# GRANT ALL PRIVILEGES ON DATABASE demo TO dbuser; +``` + +使用命令 \q 退出psql: + +``` +postgres=# \q +``` + +创建Linux普通用户,与刚才新建的数据库用户同名,如 dbuser: +``` +$ sudo adduser dbuser +$ sudo passwd dbuser +``` + +以dbuser的身份连接数据库exampledb: + +``` +$ su - dbuser + +Password: +Last login: Wed Mar 1 11:52:07 CST 2017 on pts/ + +``` + +用我们创建的用户(dbuser)登录psql + +```sql +# sudo -u dbuser psql -U dbuser -d demo +could not change directory to "/root": Permission denied +psql (12.9 (Ubuntu 12.9-0ubuntu0.20.04.1)) +Type "help" for help. + +demo=> \d + List of relations + Schema | Name | Type | Owner +--------+---------------------------+----------+---------- + public | playground | table | postgres + public | playground_1 | table | dbuser + public | playground_1_equip_id_seq | sequence | dbuser + public | playground_equip_id_seq | sequence | postgres +(4 rows) +``` + +创建表及插入数据 + +```sql +CREATE TABLE playground_test_odbc ( + equip_id serial PRIMARY KEY, + type varchar (50) NOT NULL, + color varchar (25) NOT NULL, + location varchar(25) , + install_date date +); +``` + +示例数据 + +``` +INSERT INTO playground_test_odbc (type, color, location, install_date) VALUES ('slide', 'blue', 'south', '2017-04-28'); +INSERT INTO playground_test_odbc (type, color, location, install_date) VALUES ('swing', 'yellow', 'northwest', '2018-08-16'); +``` + +执行结果 + +```sql +demo=> CREATE TABLE playground_test_odbc ( +demo(> equip_id serial PRIMARY KEY, +demo(> type varchar (50) NOT NULL, +demo(> color varchar (25) NOT NULL, +demo(> location varchar(25) , +demo(> install_date date +demo(> ); +CREATE TABLE +demo=> INSERT INTO playground_test_odbc (type, color, location, install_date) VALUES ('slide', 'blue', 'south', '2017-04-28'); +INSERT 0 1 +demo=> INSERT INTO playground_test_odbc (type, color, location, install_date) VALUES ('swing', 'yellow', 'northwest', '2018-08-16'); +INSERT 0 1 +``` + +## 4.安装Postgresql ODBC驱动 + +这里我们下载是和数据版本相对于的驱动程序 + +Postgresql ODBC驱动下载地址:https://www.postgresql.org/ftp/odbc/versions/src/ + +```shell +$> wget https://ftp.postgresql.org/pub/odbc/versions/src/psqlodbc-12.02.0000.tar.gz +$> tar zxvf psqlodbc-12.02.0000.tar.gz +$> cd psqlodbc-12.02.0000 +$> ./configure --without-libpq (注:由于本机未安装postgresql,故使用without-libpq选项) +$> ./configure +$> make +$> make install +``` + +如果在编译过程中出现下面的错误 + +``` +configure: error: libpq library version >= 9.2 is required +``` + +这是因为缺少libpq的包,需要进行安装,执行下面的命令 + +```shell +$> apt-get install libpq-dev +``` + +安装成功,默认驱动放在/usr/local/lib/psqlodbcw.so下 + +## 5.验证ODBC驱动是否成功 + +### 5.1 配置注册Postgresql ODBC驱动 + +编辑/etc/odbcinst.ini,加入下面的内容 + +```shell +[PostgreSQL] +Description = ODBC for PostgreSQL +Driver = /usr/local/lib/psqlodbcw.so +Driver64 = /usr/local/lib/psqlodbcw.so +Setup = /usr/lib/libodbc.so ##注意这里是在第二节安装的unixODBC的so文件路径 +Setup64 = /usr/lib/libodbc.so +FileUsage = 1 +``` + +### 5.2 配置PG 数据源 + +编辑/etc/odbc.ini + +加入下面内容 + +```shell +[PostgresDB] +Driver = PostgreSQL ###这里的名称和odbcinst.ini里配置的名称一致 +Description = Postgres DSN +Servername = localhost +Database = demo +Username = dbuser +Password = zhangfeng +Port = 5432 +ReadOnly = No +``` + +其他的是你的Postgresql地址及刚才创建的用户、密码、数据库、端口等 + +### 5.3 验证是否成功 + +```sql +isql -v PostgresDB dbuser zhangfeng ++---------------------------------------+ +| Connected! | +| | +| sql-statement | +| help [tablename] | +| quit | +| | ++---------------------------------------+ +SQL> +``` + +注意这里的PostgresDB是我们在odbc.ini里定义的名称,这里显示ODBC正常 + +## 6.Apache Doris PG外表验证 + +### 6.1 修改配置 + +修改BE节点conf/odbcinst.ini文件,加入刚才/etc/odbcinst.ini添加的一样内容,并删除原先的PostgreSQL配置 + +``` +[PostgreSQL] +Description = ODBC for PostgreSQL +Driver = /usr/local/lib/psqlodbcw.so +Driver64 = /usr/local/lib/psqlodbcw.so +Setup = /usr/lib/libodbc.so +Setup64 = /usr/lib/libodbc.so +FileUsage = 1 +``` + +### 6.2 验证 + +创建PG ODBC Resource + +```sql +CREATE EXTERNAL RESOURCE `pg_12` + PROPERTIES ( +"host" = "localhost", + "port" = "5432", + "user" = "dbuser", + "password" = "zhangfeng", + "database" = "demo", + "table" = "playground_test_odbc", + "driver" = "PostgreSQL", + "odbc_type" = "postgresql", + "type" = "odbc_catalog" + ); +``` + +创建ODBC外表 + +``` + CREATE EXTERNAL TABLE `playground_odbc_12` ( + equip_id int NOT NULL, + type varchar (50) NOT NULL, + color varchar (25) NOT NULL, + location varchar(25) , + install_date date +) ENGINE=ODBC +COMMENT "ODBC" +PROPERTIES ( +"odbc_catalog_resource" = "pg_12", +"database" = "demo", +"table" = "playground_test_odbc" +); +``` + +在Doris下执行查询: + +```sql +ysql> show tables; ++--------------------+ +| Tables_in_demo | ++--------------------+ +| playground_odbc_12 | +| test_odbc_5 | +| test_odbc_8 | +| test_odbc_8_0_26 | +| test_odbc_mysql | +| test_odbc_mysql_8 | ++--------------------+ +6 rows in set (0.00 sec) + +mysql> select * from playground_odbc_12; ++----------+-------+--------+-----------+--------------+ +| equip_id | type | color | location | install_date | ++----------+-------+--------+-----------+--------------+ +| 1 | slide | blue | south | 2017-04-28 | +| 2 | swing | yellow | northwest | 2018-08-16 | ++----------+-------+--------+-----------+--------------+ +2 rows in set (0.01 sec) +``` + +OK,一切正常,相对Mysql,PG的ODBC驱动更简单一些,只要你的PG版本和ODBC驱动版本对应上问题都不大。 \ No newline at end of file diff --git a/blogs/zh-CN/weekly-20220102.md b/blogs/zh-CN/weekly-20220102.md new file mode 100644 index 0000000..b23d982 --- /dev/null +++ b/blogs/zh-CN/weekly-20220102.md @@ -0,0 +1,144 @@ +--- +{ + "title": "[Doris 周报] 20211213-20220102", + "description": "Doris 周报 20211213-20220102", + "date": "2022-01-02", + "metaTitle": "[Doris 周报] 20211213-20220102", + "isArticle": true, + "language": "zh-CN", + "author": "morningman", + "layout": "Article", + "sidebar": false +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +## 统计数据 + +共 37 位作者提交了 82 个 Commit 。感谢以下作者的贡献(Github ID,按字母序): + +BiteTheDDDDt, CalvinKirs, EmmyMiao87, GoGoWen, HappenLee, Henry2SS, JNSimba, aihai, caiconghui, dh-cloud, gaodayue, hf200012, jackwener, liutang123, luozenglin, luzhijing, morningman, pengxiangyu, qidaye, qzsee, spaces-X, tianhui5, tinkerrrr, wangshuo128, wayofeng, weizuo93, whutpencil, xiedeyantu, xinyiZzz, xtr1993, xuzifu666, yangzhg, yjant, zbtzbtzbt, zenoyang, zhangstar333, zhengshiJ + +最近 3 周,共修改新增代码行 15882 ,删除代码行 6966。 + +## 主要进展 + +### 新增功能 + +* https://github.com/apache/incubator-doris/pull/7276 + + 支持直接返回 HLL/BITMAP 列的二进制数据,以供外部解析使用。 + +* https://github.com/apache/incubator-doris/pull/7255 + + 支持 Lateral View 列转行功能。方便将 bitmap、stirng 或 json array 数据展开成多行,以供后续处理。 + +* https://github.com/apache/incubator-doris/pull/7379 + + `bitmap_union` 和 `bitmap_intersect` 支持多个 bitmap 参数。以提高多个 bitmap 求交集和并集的效率。 + +* https://github.com/apache/incubator-doris/pull/7464 + + 支持国密算法函数 SM3/SM4。 + +### Work-in-Progess + +* https://github.com/apache/incubator-doris/pull/7098 + + 支持冷数据转储到对象存储(开发中)。 + +### Bug 修复 + +* https://github.com/apache/incubator-doris/pull/7383 + + 修复 Segment Cache 某些情况下没有正确处理过期缓存的问题。 + +* https://github.com/apache/incubator-doris/pull/7373 + + 修复 BE 侧常量折叠代码的若干Bug。 + +* https://github.com/apache/incubator-doris/pull/7371 + + 修复 Broker Load 调度逻辑在某些情况下可能导致任务不被调度的问题。 + +* https://github.com/apache/incubator-doris/pull/7375 + + 修复部分逻辑视图查询导致 SQL Cache无法更新的问题。 + +* https://github.com/apache/incubator-doris/pull/7407 + + 优化 SQL block rule 的逻辑,使其仅对 select 语句生效。 + +* https://github.com/apache/incubator-doris/pull/7421 + + 修复 round() 函数结果不正确的问题。 + +* https://github.com/apache/incubator-doris/pull/7362 + + 修复部分情况下,物化视图改写 count 函数时逻辑错误的问题。 + +* https://github.com/apache/incubator-doris/pull/7495 + + 修复在表名大小写不敏感设定环境下,部分查询中别名解析错误的问题。 + +* https://github.com/apache/incubator-doris/pull/7440 + + 修复 bloom filter 处理空字符串错误的问题。 + +* https://github.com/apache/incubator-doris/pull/7411 + + 修复部分情况下 Grouping Set 功能无法正确处理表达式的问题。 + +### 功能改进 + +* https://github.com/apache/incubator-doris/pull/7232 + + Flink-Connector 支持并发读取 Doris 数据。 + +* https://github.com/apache/incubator-doris/pull/7348 + + 优化 Unique Key 表在 Reader 层的读取效率。 + +* https://github.com/apache/incubator-doris/pull/7454 + + 优化当导入大量不合规数据时,BE 的内存开销。 + +* https://github.com/apache/incubator-doris/pull/7096 + + 查询优化器支持更多谓词传递优化。 + +* https://github.com/apache/incubator-doris/pull/7435 + + 优化 Flink-Connector 的 BE 选取逻辑,仅选择存活的 BE 节点。 + +* https://github.com/apache/incubator-doris/pull/7493 + + 修复 brpc 的一些默认参数以降低 BRPC 网络传输错误的概率。 + +* https://github.com/apache/incubator-doris/pull/7434 + + 重构查询优化器分区裁剪逻辑,以支持更多谓词条件的分区裁剪能力。 + +### 其他 + +* https://github.com/apache/incubator-doris/pull/7536 + + 升级 Log4j 到 2.7.1 已修复安全漏洞 CVE-2021-44832 --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org