[
https://issues.apache.org/jira/browse/HIVE-23786?focusedWorklogId=455856&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-455856
]
ASF GitHub Bot logged work on HIVE-23786:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 08/Jul/20 05:26
Start Date: 08/Jul/20 05:26
Worklog Time Spent: 10m
Work Description: pvary commented on a change in pull request #1221:
URL: https://github.com/apache/hive/pull/1221#discussion_r451290794
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java
##########
@@ -85,38 +97,268 @@ public final void onEvent(PreEventContext preEventContext)
throws MetaException,
LOG.debug("==> HiveMetaStoreAuthorizer.onEvent(): EventType=" +
preEventContext.getEventType());
}
- HiveMetaStoreAuthzInfo authzContext = buildAuthzContext(preEventContext);
+ try {
+ HiveAuthorizer hiveAuthorizer = createHiveMetaStoreAuthorizer();
+ if (!skipAuthorization()) {
+ HiveMetaStoreAuthzInfo authzContext =
buildAuthzContext(preEventContext);
+ checkPrivileges(authzContext, hiveAuthorizer);
+ }
+ } catch (Exception e) {
+ LOG.error("HiveMetaStoreAuthorizer.onEvent(): failed", e);
+ throw new MetaException(e.getMessage());
+ }
- if (!skipAuthorization(authzContext)) {
- try {
- HiveConf hiveConf = new
HiveConf(super.getConf(), HiveConf.class);
- HiveAuthorizerFactory authorizerFactory =
HiveUtils.getAuthorizerFactory(hiveConf,
HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveMetaStoreAuthorizer.onEvent(): EventType=" +
preEventContext.getEventType());
+ }
+ }
- if (authorizerFactory != null) {
- HiveMetastoreAuthenticationProvider authenticator =
tAuthenticator.get();
+ @Override
+ public final List<String> filterDatabases(List<String> list) throws
MetaException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("HiveMetaStoreAuthorizer.filterDatabases()");
+ }
- authenticator.setConf(hiveConf);
+ if (list == null) {
+ return Collections.emptyList();
+ }
- HiveAuthzSessionContext.Builder authzContextBuilder = new
HiveAuthzSessionContext.Builder();
+ DatabaseFilterContext databaseFilterContext = new
DatabaseFilterContext(list);
+ HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo =
databaseFilterContext.getAuthzContext();
+ List<String> filteredDatabases =
filterDatabaseObjects(hiveMetaStoreAuthzInfo);
+ if (CollectionUtils.isEmpty(filteredDatabases)) {
+ filteredDatabases = Collections.emptyList();
+ }
-
authzContextBuilder.setClientType(HiveAuthzSessionContext.CLIENT_TYPE.HIVEMETASTORE);
- authzContextBuilder.setSessionString("HiveMetaStore");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("HiveMetaStoreAuthorizer.filterDatabases() :" +
filteredDatabases);
+ }
+ return filteredDatabases ;
+ }
- HiveAuthzSessionContext authzSessionContext =
authzContextBuilder.build();
+ @Override
+ public final Database filterDatabase(Database database) throws
MetaException, NoSuchObjectException {
+ if (database != null) {
+ String dbName = database.getName();
+ List<String> databases =
filterDatabases(Collections.singletonList(dbName));
+ if (databases.isEmpty()) {
+ throw new NoSuchObjectException(String.format("Database %s does not
exist", dbName));
+ }
+ }
+ return database;
+ }
+
+ @Override
+ public final List<String> filterTableNames(String s, String s1, List<String>
list) throws MetaException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> HiveMetaStoreAuthorizer.filterTableNames()");
+ }
+ List<String> filteredTableNames = null;
+ if (list != null) {
+ String dbName = getDBName(s1);
+ TableFilterContext tableFilterContext = new
TableFilterContext(dbName, list);
+ HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo =
tableFilterContext.getAuthzContext();
+ filteredTableNames = filterTableNames(hiveMetaStoreAuthzInfo, dbName,
list);
+ if (CollectionUtils.isEmpty(filteredTableNames)) {
+ filteredTableNames = Collections.emptyList();
+ }
+ }
- HiveAuthorizer hiveAuthorizer =
authorizerFactory.createHiveAuthorizer(new HiveMetastoreClientFactoryImpl(),
hiveConf, authenticator, authzSessionContext);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveMetaStoreAuthorizer.filterTableNames() : " +
filteredTableNames);
+ }
- checkPrivileges(authzContext, hiveAuthorizer);
- }
- } catch (Exception e) {
- LOG.error("HiveMetaStoreAuthorizer.onEvent(): failed", e);
- throw new MetaException(e.getMessage());
+ return filteredTableNames;
+ }
+
+ @Override
+ public final Table filterTable(Table table) throws MetaException,
NoSuchObjectException {
+ if (table != null) {
+ List<Table> tables = filterTables(Collections.singletonList(table));
+ if (tables.isEmpty()) {
+ throw new NoSuchObjectException(String.format("Database %s does not
exist", table.getTableName()));
}
}
+ return table;
+ }
+ @Override
+ public final List<Table> filterTables(List<Table> list) throws MetaException
{
if (LOG.isDebugEnabled()) {
- LOG.debug("<== HiveMetaStoreAuthorizer.onEvent(): EventType=" +
preEventContext.getEventType());
+ LOG.debug("==> HiveMetaStoreAuthorizer.filterTables()");
+ }
+
+ List<Table> filteredTables = null;
+
+ if (list != null) {
+ TableFilterContext tableFilterContext = new
TableFilterContext(list);
+ HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo =
tableFilterContext.getAuthzContext();
+ filteredTables = filterTableObjects(hiveMetaStoreAuthzInfo, list);
+ if (CollectionUtils.isEmpty(filteredTables)) {
+ filteredTables = Collections.emptyList();
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveMetaStoreAuthorizer.filterTables(): " +
filteredTables);
+ }
+ return filteredTables;
+ }
+
+ @Override
+ public final Catalog filterCatalog(Catalog catalog) throws MetaException {
+ return catalog;
+ }
+
+ @Override
+ public final List<String> filterCatalogs(List<String> catalogs) throws
MetaException {
+ return catalogs;
+ }
+
+ @Override
+ public final List<TableMeta> filterTableMetas(String catName,String
dbName,List<TableMeta> tableMetas) throws MetaException {
+ return tableMetas;
+ }
+
+ @Override
+ public final List<Partition> filterPartitions(List<Partition> list) throws
MetaException {
+ return list;
+ }
+
+ @Override
+ public final List<PartitionSpec> filterPartitionSpecs(List<PartitionSpec>
list) throws MetaException {
+ return list;
+ }
+
+ @Override
+ public final Partition filterPartition(Partition partition) throws
MetaException, NoSuchObjectException {
+ return partition;
+ }
+
+ @Override
+ public final List<String> filterPartitionNames(String s, String s1, String
s2, List<String> list) throws MetaException {
+ return list;
+ }
+
+ private List<String> filterDatabaseObjects(HiveMetaStoreAuthzInfo
hiveMetaStoreAuthzInfo) throws MetaException {
+ List<String> ret = null;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> HiveMetaStoreAuthorizer.filterDatabaseObjects()");
+ }
+
+ try {
+ HiveAuthorizer hiveAuthorizer =
createHiveMetaStoreAuthorizer();
+ List<HivePrivilegeObject> hivePrivilegeObjects =
hiveMetaStoreAuthzInfo.getInputHObjs();
+ HiveAuthzContext hiveAuthzContext =
hiveMetaStoreAuthzInfo.getHiveAuthzContext();
+ List<HivePrivilegeObject> filteredHivePrivilegeObjects =
hiveAuthorizer.filterListCmdObjects(hivePrivilegeObjects, hiveAuthzContext);
+ if (CollectionUtils.isNotEmpty(filteredHivePrivilegeObjects)) {
+ ret = getFilterDatabaseList(filteredHivePrivilegeObjects);
+ }
+ } catch (Exception e) {
+ throw new MetaException("Error in
HiveMetaStoreAuthorizer.filterDatabase()" + e.getMessage());
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveMetaStoreAuthorizer.filterDatabaseObjects() :" + ret
);
}
+ return ret;
+ }
+
+ private List<Table> filterTableObjects(HiveMetaStoreAuthzInfo
hiveMetaStoreAuthzInfo, List<Table> tableList) throws MetaException {
+ List<Table> ret = null;
+
+ try {
+ HiveAuthorizer hiveAuthorizer =
createHiveMetaStoreAuthorizer();
+ List<HivePrivilegeObject> hivePrivilegeObjects =
hiveMetaStoreAuthzInfo.getInputHObjs();
+ HiveAuthzContext hiveAuthzContext =
hiveMetaStoreAuthzInfo.getHiveAuthzContext();
+ List<HivePrivilegeObject> filteredHivePrivilegeObjects =
hiveAuthorizer.filterListCmdObjects(hivePrivilegeObjects, hiveAuthzContext);
+ if (CollectionUtils.isNotEmpty(filteredHivePrivilegeObjects)) {
+ ret = getFilteredTableList(filteredHivePrivilegeObjects, tableList);
+ }
+ } catch (Exception e) {
+ throw new MetaException("Error in
HiveMetaStoreAuthorizer.filterTables()" + e.getMessage());
+ }
+ return ret;
+ }
+
+ private List<String> getFilterDatabaseList(List<HivePrivilegeObject>
hivePrivilegeObjects) {
+ List<String> ret = new ArrayList<>();
+ for(HivePrivilegeObject hivePrivilegeObject:hivePrivilegeObjects) {
+ String dbName = hivePrivilegeObject.getDbname();
+ ret.add(dbName);
+ }
+ return ret;
+ }
+
+ private List<Table> getFilteredTableList(List<HivePrivilegeObject>
hivePrivilegeObjects, List<Table> tableList) {
+ List<Table> ret = new ArrayList<>();
+ for(HivePrivilegeObject hivePrivilegeObject:hivePrivilegeObjects) {
+ String dbName = hivePrivilegeObject.getDbname();
+ String tblName = hivePrivilegeObject.getObjectName();
+ Table table = getFilteredTable(dbName,tblName,tableList);
+ if (table != null) {
+ ret.add(table);
+ }
+ }
+ return ret;
+ }
+
+ private Table getFilteredTable(String dbName, String tblName, List<Table>
tableList) {
+ Table ret = null;
+ for (Table table: tableList) {
+ String databaseName = table.getDbName();
+ String tableName = table.getTableName();
+ if (dbName.equals(databaseName) && tblName.equals(tableName)) {
+ ret = table;
+ break;
+ }
+ }
+ return ret;
+ }
+
+ private List<String> filterTableNames(HiveMetaStoreAuthzInfo
hiveMetaStoreAuthzInfo, String dbName, List<String> tableNames) throws
MetaException {
+ List<String> ret = null;
+
+ try {
+ HiveAuthorizer hiveAuthorizer =
createHiveMetaStoreAuthorizer();
+ List<HivePrivilegeObject> hivePrivilegeObjects =
hiveMetaStoreAuthzInfo.getInputHObjs();
+ HiveAuthzContext hiveAuthzContext =
hiveMetaStoreAuthzInfo.getHiveAuthzContext();
+ List<HivePrivilegeObject> filteredHivePrivilegeObjects =
hiveAuthorizer.filterListCmdObjects(hivePrivilegeObjects, hiveAuthzContext);
Review comment:
How big is this list in a worst case? I would be very much afraid to
keep all this stuff in memory. I have seen customers with ~1 million tables.
Would it fit into the memory? Even for multiple parallel requests?
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java
##########
@@ -85,38 +97,268 @@ public final void onEvent(PreEventContext preEventContext)
throws MetaException,
LOG.debug("==> HiveMetaStoreAuthorizer.onEvent(): EventType=" +
preEventContext.getEventType());
}
- HiveMetaStoreAuthzInfo authzContext = buildAuthzContext(preEventContext);
+ try {
+ HiveAuthorizer hiveAuthorizer = createHiveMetaStoreAuthorizer();
+ if (!skipAuthorization()) {
+ HiveMetaStoreAuthzInfo authzContext =
buildAuthzContext(preEventContext);
+ checkPrivileges(authzContext, hiveAuthorizer);
+ }
+ } catch (Exception e) {
+ LOG.error("HiveMetaStoreAuthorizer.onEvent(): failed", e);
+ throw new MetaException(e.getMessage());
+ }
- if (!skipAuthorization(authzContext)) {
- try {
- HiveConf hiveConf = new
HiveConf(super.getConf(), HiveConf.class);
- HiveAuthorizerFactory authorizerFactory =
HiveUtils.getAuthorizerFactory(hiveConf,
HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveMetaStoreAuthorizer.onEvent(): EventType=" +
preEventContext.getEventType());
+ }
+ }
- if (authorizerFactory != null) {
- HiveMetastoreAuthenticationProvider authenticator =
tAuthenticator.get();
+ @Override
+ public final List<String> filterDatabases(List<String> list) throws
MetaException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("HiveMetaStoreAuthorizer.filterDatabases()");
+ }
- authenticator.setConf(hiveConf);
+ if (list == null) {
+ return Collections.emptyList();
+ }
- HiveAuthzSessionContext.Builder authzContextBuilder = new
HiveAuthzSessionContext.Builder();
+ DatabaseFilterContext databaseFilterContext = new
DatabaseFilterContext(list);
+ HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo =
databaseFilterContext.getAuthzContext();
+ List<String> filteredDatabases =
filterDatabaseObjects(hiveMetaStoreAuthzInfo);
+ if (CollectionUtils.isEmpty(filteredDatabases)) {
+ filteredDatabases = Collections.emptyList();
+ }
-
authzContextBuilder.setClientType(HiveAuthzSessionContext.CLIENT_TYPE.HIVEMETASTORE);
- authzContextBuilder.setSessionString("HiveMetaStore");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("HiveMetaStoreAuthorizer.filterDatabases() :" +
filteredDatabases);
+ }
+ return filteredDatabases ;
+ }
- HiveAuthzSessionContext authzSessionContext =
authzContextBuilder.build();
+ @Override
+ public final Database filterDatabase(Database database) throws
MetaException, NoSuchObjectException {
+ if (database != null) {
+ String dbName = database.getName();
+ List<String> databases =
filterDatabases(Collections.singletonList(dbName));
+ if (databases.isEmpty()) {
+ throw new NoSuchObjectException(String.format("Database %s does not
exist", dbName));
+ }
+ }
+ return database;
+ }
+
+ @Override
+ public final List<String> filterTableNames(String s, String s1, List<String>
list) throws MetaException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> HiveMetaStoreAuthorizer.filterTableNames()");
+ }
+ List<String> filteredTableNames = null;
+ if (list != null) {
+ String dbName = getDBName(s1);
+ TableFilterContext tableFilterContext = new
TableFilterContext(dbName, list);
+ HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo =
tableFilterContext.getAuthzContext();
+ filteredTableNames = filterTableNames(hiveMetaStoreAuthzInfo, dbName,
list);
+ if (CollectionUtils.isEmpty(filteredTableNames)) {
+ filteredTableNames = Collections.emptyList();
+ }
+ }
- HiveAuthorizer hiveAuthorizer =
authorizerFactory.createHiveAuthorizer(new HiveMetastoreClientFactoryImpl(),
hiveConf, authenticator, authzSessionContext);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveMetaStoreAuthorizer.filterTableNames() : " +
filteredTableNames);
+ }
- checkPrivileges(authzContext, hiveAuthorizer);
- }
- } catch (Exception e) {
- LOG.error("HiveMetaStoreAuthorizer.onEvent(): failed", e);
- throw new MetaException(e.getMessage());
+ return filteredTableNames;
+ }
+
+ @Override
+ public final Table filterTable(Table table) throws MetaException,
NoSuchObjectException {
+ if (table != null) {
+ List<Table> tables = filterTables(Collections.singletonList(table));
+ if (tables.isEmpty()) {
+ throw new NoSuchObjectException(String.format("Database %s does not
exist", table.getTableName()));
}
}
+ return table;
+ }
+ @Override
+ public final List<Table> filterTables(List<Table> list) throws MetaException
{
if (LOG.isDebugEnabled()) {
- LOG.debug("<== HiveMetaStoreAuthorizer.onEvent(): EventType=" +
preEventContext.getEventType());
+ LOG.debug("==> HiveMetaStoreAuthorizer.filterTables()");
+ }
+
+ List<Table> filteredTables = null;
+
+ if (list != null) {
+ TableFilterContext tableFilterContext = new
TableFilterContext(list);
+ HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo =
tableFilterContext.getAuthzContext();
+ filteredTables = filterTableObjects(hiveMetaStoreAuthzInfo, list);
+ if (CollectionUtils.isEmpty(filteredTables)) {
+ filteredTables = Collections.emptyList();
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveMetaStoreAuthorizer.filterTables(): " +
filteredTables);
+ }
+ return filteredTables;
+ }
+
+ @Override
+ public final Catalog filterCatalog(Catalog catalog) throws MetaException {
+ return catalog;
+ }
+
+ @Override
+ public final List<String> filterCatalogs(List<String> catalogs) throws
MetaException {
+ return catalogs;
+ }
+
+ @Override
+ public final List<TableMeta> filterTableMetas(String catName,String
dbName,List<TableMeta> tableMetas) throws MetaException {
+ return tableMetas;
+ }
+
+ @Override
+ public final List<Partition> filterPartitions(List<Partition> list) throws
MetaException {
+ return list;
+ }
+
+ @Override
+ public final List<PartitionSpec> filterPartitionSpecs(List<PartitionSpec>
list) throws MetaException {
+ return list;
+ }
+
+ @Override
+ public final Partition filterPartition(Partition partition) throws
MetaException, NoSuchObjectException {
+ return partition;
+ }
+
+ @Override
+ public final List<String> filterPartitionNames(String s, String s1, String
s2, List<String> list) throws MetaException {
+ return list;
+ }
+
+ private List<String> filterDatabaseObjects(HiveMetaStoreAuthzInfo
hiveMetaStoreAuthzInfo) throws MetaException {
+ List<String> ret = null;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> HiveMetaStoreAuthorizer.filterDatabaseObjects()");
+ }
+
+ try {
+ HiveAuthorizer hiveAuthorizer =
createHiveMetaStoreAuthorizer();
+ List<HivePrivilegeObject> hivePrivilegeObjects =
hiveMetaStoreAuthzInfo.getInputHObjs();
+ HiveAuthzContext hiveAuthzContext =
hiveMetaStoreAuthzInfo.getHiveAuthzContext();
+ List<HivePrivilegeObject> filteredHivePrivilegeObjects =
hiveAuthorizer.filterListCmdObjects(hivePrivilegeObjects, hiveAuthzContext);
+ if (CollectionUtils.isNotEmpty(filteredHivePrivilegeObjects)) {
+ ret = getFilterDatabaseList(filteredHivePrivilegeObjects);
+ }
+ } catch (Exception e) {
+ throw new MetaException("Error in
HiveMetaStoreAuthorizer.filterDatabase()" + e.getMessage());
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveMetaStoreAuthorizer.filterDatabaseObjects() :" + ret
);
}
+ return ret;
+ }
+
+ private List<Table> filterTableObjects(HiveMetaStoreAuthzInfo
hiveMetaStoreAuthzInfo, List<Table> tableList) throws MetaException {
+ List<Table> ret = null;
+
+ try {
+ HiveAuthorizer hiveAuthorizer =
createHiveMetaStoreAuthorizer();
+ List<HivePrivilegeObject> hivePrivilegeObjects =
hiveMetaStoreAuthzInfo.getInputHObjs();
+ HiveAuthzContext hiveAuthzContext =
hiveMetaStoreAuthzInfo.getHiveAuthzContext();
+ List<HivePrivilegeObject> filteredHivePrivilegeObjects =
hiveAuthorizer.filterListCmdObjects(hivePrivilegeObjects, hiveAuthzContext);
+ if (CollectionUtils.isNotEmpty(filteredHivePrivilegeObjects)) {
+ ret = getFilteredTableList(filteredHivePrivilegeObjects, tableList);
+ }
+ } catch (Exception e) {
+ throw new MetaException("Error in
HiveMetaStoreAuthorizer.filterTables()" + e.getMessage());
+ }
+ return ret;
+ }
+
+ private List<String> getFilterDatabaseList(List<HivePrivilegeObject>
hivePrivilegeObjects) {
+ List<String> ret = new ArrayList<>();
+ for(HivePrivilegeObject hivePrivilegeObject:hivePrivilegeObjects) {
+ String dbName = hivePrivilegeObject.getDbname();
+ ret.add(dbName);
+ }
+ return ret;
+ }
+
+ private List<Table> getFilteredTableList(List<HivePrivilegeObject>
hivePrivilegeObjects, List<Table> tableList) {
+ List<Table> ret = new ArrayList<>();
+ for(HivePrivilegeObject hivePrivilegeObject:hivePrivilegeObjects) {
+ String dbName = hivePrivilegeObject.getDbname();
+ String tblName = hivePrivilegeObject.getObjectName();
+ Table table = getFilteredTable(dbName,tblName,tableList);
Review comment:
Spaces
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 455856)
Time Spent: 2h 10m (was: 2h)
> HMS Server side filter
> ----------------------
>
> Key: HIVE-23786
> URL: https://issues.apache.org/jira/browse/HIVE-23786
> Project: Hive
> Issue Type: Improvement
> Reporter: Sam An
> Assignee: Sam An
> Priority: Major
> Labels: pull-request-available
> Time Spent: 2h 10m
> Remaining Estimate: 0h
>
> HMS server side filter of results based on authorization.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
