Skip to content

HIVE-29241: Distinguish the default location of the database by catalog #6267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zhangbutao wants to merge 6 commits into
base: master
Choose a base branch
from
Open

HIVE-29241: Distinguish the default location of the database by catalog #6267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4991c8d
Distinguish the location of the database by catalog
zhangbutao Dec 25, 2025
49c343a
Fix Copilot comments
zhangbutao Jan 15, 2026
d9157b3
Fix null catalog
zhangbutao Jan 16, 2026
fa2afa9
Added system property of non-default catalog warehouse location for j…
zhangbutao Jan 16, 2026
b177f8a
Fix tests
zhangbutao Jan 17, 2026
2b8130d
Accept some Copilot advice
zhangbutao Jan 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -922,6 +922,15 @@ public static enum ConfVars {
"Default location for external tables created in the warehouse. " +
"If not set or null, then the normal warehouse location will be used as the default location."),

METASTORE_CATALOG_WAREHOUSE("hive.metastore.warehouse.catalog.dir", "/user/hive/catalog/warehouse",
"Base location for databases in non-default catalogs. If the name of the catalog to which the db belongs is testcat, " +
"then the default path prefix for the db is /user/hive/catalog/warehouse/testcat"),

HIVE_METASTORE_CATALOG_WAREHOUSE_EXTERNAL("hive.metastore.warehouse.catalog.external.dir", null,
"Base location for external tables created in the warehouse of non-default catalogs. " +
"If not set or null, then the normal warehouse location (MetastoreConf.METASTORE_CATALOG_WAREHOUSE) " +
"will be used as the default location."),

/**
* @deprecated Use MetastoreConf.THRIFT_URIS
*/
Expand Down
5 changes: 5 additions & 0 deletions itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,7 @@ private void setFsRelatedProperties(HiveConf conf, boolean isLocalFs, FileSystem
// Different paths if running locally vs a remote fileSystem. Ideally this difference should not
// exist.
Path warehousePath;
Path warehouseCatPath;
Path jarPath;
Path userInstallPath;
if (isLocalFs) {
Expand All @@ -554,23 +555,27 @@ private void setFsRelatedProperties(HiveConf conf, boolean isLocalFs, FileSystem
// Create a fake fs root for local fs
Path localFsRoot = new Path(path, "localfs");
warehousePath = new Path(localFsRoot, "warehouse");
warehouseCatPath = new Path(localFsRoot, "catalog");
jarPath = new Path(localFsRoot, "jar");
userInstallPath = new Path(localFsRoot, "user_install");
} else {
// TODO Why is this changed from the default in hive-conf?
warehousePath = new Path(fsUriString, "/build/ql/test/data/warehouse/");
warehouseCatPath = new Path(fsUriString, "/build/ql/test/data/catalog/");
jarPath = new Path(new Path(fsUriString, "/user"), "hive");
userInstallPath = new Path(fsUriString, "/user");
}

warehousePath = fs.makeQualified(warehousePath);
warehouseCatPath = fs.makeQualified(warehouseCatPath);
jarPath = fs.makeQualified(jarPath);
userInstallPath = fs.makeQualified(userInstallPath);

conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsUriString);

// Remote dirs
conf.setVar(ConfVars.METASTORE_WAREHOUSE, warehousePath.toString());
conf.setVar(ConfVars.METASTORE_CATALOG_WAREHOUSE, warehouseCatPath.toString());
conf.setVar(ConfVars.HIVE_JAR_DIRECTORY, jarPath.toString());
conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, userInstallPath.toString());
// ConfVars.SCRATCH_DIR - {test.tmp.dir}/scratchdir
Expand Down
6 changes: 3 additions & 3 deletions parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1131,10 +1131,10 @@ createCatalogStatement
: KW_CREATE KW_CATALOG
ifNotExists?
name=identifier
catLocation
catLocation?
catalogComment?
(KW_PROPERTIES catprops=properties)?
-> ^(TOK_CREATECATALOG $name catLocation ifNotExists? catalogComment? $catprops?)
KW_PROPERTIES catprops=properties
-> ^(TOK_CREATECATALOG $name catLocation? ifNotExists? catalogComment? $catprops)
;

catLocation
Expand Down
24 changes: 21 additions & 3 deletions ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/create/CreateCatalogAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

package org.apache.hadoop.hive.ql.ddl.catalog.create;

import com.google.common.base.Strings;
import org.apache.hadoop.hive.metastore.CatalogUtil;
import org.apache.hadoop.hive.metastore.api.Catalog;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory;
Expand All @@ -43,16 +45,19 @@ public CreateCatalogAnalyzer(QueryState queryState) throws SemanticException {
@Override
public void analyzeInternal(ASTNode root) throws SemanticException {
String catalogName = unescapeIdentifier(root.getChild(0).getText());
String locationUrl = unescapeSQLString(root.getChild(1).getChild(0).getText());
outputs.add(toWriteEntity(locationUrl));

String locationUrl = null;
boolean ifNotExists = false;
String comment = null;
Map<String, String> props = null;

for (int i = 2; i < root.getChildCount(); i++) {
for (int i = 1; i < root.getChildCount(); i++) {
ASTNode childNode = (ASTNode) root.getChild(i);
switch (childNode.getToken().getType()) {
case HiveParser.TOK_CATALOGLOCATION:
locationUrl = unescapeSQLString(childNode.getChild(0).getText());
outputs.add(toWriteEntity(locationUrl));
break;
case HiveParser.TOK_IFNOTEXISTS:
ifNotExists = true;
break;
Expand All @@ -67,10 +72,23 @@ public void analyzeInternal(ASTNode root) throws SemanticException {
}
}

assert props != null;
checkCatalogType(props);

CreateCatalogDesc desc = new CreateCatalogDesc(catalogName, comment, locationUrl, ifNotExists, props);
Catalog catalog = new Catalog(catalogName, locationUrl);

rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)));
outputs.add(new WriteEntity(catalog, WriteEntity.WriteType.DDL_NO_LOCK));
}

private static void checkCatalogType(Map<String, String> props) throws SemanticException {
String catalogType = props.get("type");
if (Strings.isNullOrEmpty(catalogType)) {
throw new SemanticException("'type' can not be null or empty");
}
if (!CatalogUtil.isValidCatalogType(catalogType)) {
throw new SemanticException(String.format("type '%s' is not valid", catalogType));
}
}
}
7 changes: 6 additions & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/create/CreateCatalogOperation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.Catalog;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.ddl.DDLOperation;
import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
Expand All @@ -37,7 +38,11 @@ public CreateCatalogOperation(DDLOperationContext context, CreateCatalogDesc des

@Override
public int execute() throws Exception {
Catalog catalog = new Catalog(desc.getName(), desc.getLocationUri());
String catLocationUri = Optional.ofNullable(desc.getLocationUri())
.orElse(MetastoreConf.getVar(context.getConf(),
MetastoreConf.ConfVars.WAREHOUSE_CATALOG) + "/" + desc.getName());

Catalog catalog = new Catalog(desc.getName(), catLocationUri);
catalog.setDescription(desc.getComment());
Optional.ofNullable(desc.getCatlogProperties())
.ifPresent(catalog::setParameters);
Expand Down
10 changes: 5 additions & 5 deletions ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,21 @@
package org.apache.hadoop.hive.ql.ddl.database.create;

import java.util.Map;
import java.util.Optional;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hive.metastore.api.DataConnector;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.DatabaseType;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.ddl.DDLUtils;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.HiveParser;
Expand All @@ -51,10 +52,9 @@ public CreateDatabaseAnalyzer(QueryState queryState) throws SemanticException {
@Override
public void analyzeInternal(ASTNode root) throws SemanticException {
Pair<String, String> catDbNamePair = DDLUtils.getCatDbNamePair((ASTNode) root.getChild(0));
String catalogName = catDbNamePair.getLeft();
if (catalogName != null && getCatalog(catalogName) == null) {
throw new SemanticException(ErrorMsg.CATALOG_NOT_EXISTS, catalogName);
}
String catalogName = Optional.ofNullable(catDbNamePair.getLeft())
.orElse(HiveUtils.getCurrentCatalogOrDefault(conf));

String databaseName = catDbNamePair.getRight();

boolean ifNotExists = false;
Expand Down
62 changes: 45 additions & 17 deletions ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseOperation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,12 @@ public int execute() throws HiveException {
if (desc.getManagedLocationUri() != null) {
database.setManagedLocationUri(desc.getManagedLocationUri());
}
makeLocationQualified(database); // TODO catalog. Add catalog prefix for db location. Depend on HIVE-29241.
makeLocationQualified(database);
if (database.getLocationUri().equalsIgnoreCase(database.getManagedLocationUri())) {
throw new HiveException("Managed and external locations for database cannot be the same");
}
} else if (desc.getDatabaseType() == DatabaseType.REMOTE) {
makeLocationQualified(database); // TODO catalog. Add catalog prefix for db location. Depend on HIVE-29241.
makeLocationQualified(database);
database.setConnector_name(desc.getConnectorName());
database.setRemote_dbname(desc.getRemoteDbName());
} else { // should never be here
Expand All @@ -81,34 +81,62 @@ public int execute() throws HiveException {
}

private void makeLocationQualified(Database database) throws HiveException {
String catalogName = database.getCatalogName().toLowerCase();
String dbName = database.getName().toLowerCase();
boolean isDefaultCatalog = Warehouse.DEFAULT_CATALOG_NAME.equalsIgnoreCase(catalogName);

// -------- External location --------
if (database.isSetLocationUri()) {
database.setLocationUri(Utilities.getQualifiedPath(context.getConf(), new Path(database.getLocationUri())));
} else {
// Location is not set we utilize WAREHOUSE_EXTERNAL together with database name
String rootDir = MetastoreConf.getVar(context.getConf(), MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL);
if (rootDir == null || rootDir.trim().isEmpty()) {
// Fallback plan
LOG.warn(String.format(
"%s is not set, falling back to %s. This could cause external tables to use to managed tablespace.",
MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname(), MetastoreConf.ConfVars.WAREHOUSE.getVarname()));
rootDir = MetastoreConf.getVar(context.getConf(), MetastoreConf.ConfVars.WAREHOUSE);
}
Path path = new Path(rootDir, database.getName().toLowerCase() + DATABASE_PATH_SUFFIX);
String qualifiedPath = Utilities.getQualifiedPath(context.getConf(), path);
database.setLocationUri(qualifiedPath);
String rootDir = getExternalRootDir(isDefaultCatalog);
Path path = buildDbPath(rootDir, catalogName, dbName, isDefaultCatalog);
database.setLocationUri(Utilities.getQualifiedPath(context.getConf(), path));
}

// -------- Managed location --------
if (database.isSetManagedLocationUri()) {
database.setManagedLocationUri(Utilities.getQualifiedPath(context.getConf(),
new Path(database.getManagedLocationUri())));
} else {
// ManagedLocation is not set we utilize WAREHOUSE together with database name
String rootDir = MetastoreConf.getVar(context.getConf(), MetastoreConf.ConfVars.WAREHOUSE);
Path path = new Path(rootDir, database.getName().toLowerCase() + DATABASE_PATH_SUFFIX);
String rootDir = MetastoreConf.getVar(
context.getConf(),
isDefaultCatalog
? MetastoreConf.ConfVars.WAREHOUSE
: MetastoreConf.ConfVars.WAREHOUSE_CATALOG
);

Path path = buildDbPath(rootDir, catalogName, dbName, isDefaultCatalog);
String qualifiedPath = Utilities.getQualifiedPath(context.getConf(), path);
if (!qualifiedPath.equals(database.getLocationUri())) {
database.setManagedLocationUri(qualifiedPath);
}
}
}

private Path buildDbPath(String rootDir, String catalogName, String dbName, boolean isDefaultCatalog) {
return isDefaultCatalog
? new Path(rootDir, dbName + DATABASE_PATH_SUFFIX)
: new Path(rootDir + "/" + catalogName, dbName + DATABASE_PATH_SUFFIX);
}

private String getExternalRootDir(boolean isDefaultCatalog) {
MetastoreConf.ConfVars externalVar = isDefaultCatalog
? MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL
: MetastoreConf.ConfVars.WAREHOUSE_CATALOG_EXTERNAL;

String rootDir = MetastoreConf.getVar(context.getConf(), externalVar);
if (rootDir != null && !rootDir.trim().isEmpty()) {
return rootDir;
}

MetastoreConf.ConfVars fallbackVar = isDefaultCatalog
? MetastoreConf.ConfVars.WAREHOUSE
: MetastoreConf.ConfVars.WAREHOUSE_CATALOG;

LOG.warn("{} is not set, falling back to {}. This could cause external tables to use managed tablespace.",
externalVar.getVarname(), fallbackVar.getVarname());

return MetastoreConf.getVar(context.getConf(), fallbackVar);
}
}
31 changes: 26 additions & 5 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.repl.ReplConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
Expand Down Expand Up @@ -388,14 +389,34 @@ public static void createDbExportDump(FileSystem fs, Path metadataPath, Database

private static void updateIfCustomDbLocations(Database database, Configuration conf) throws SemanticException {
try {
String whLocatoion = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL,
MetastoreConf.getVar(conf, MetastoreConf.ConfVars.WAREHOUSE));
Path dbDerivedLoc = new Path(whLocatoion, database.getName().toLowerCase() + DATABASE_PATH_SUFFIX);
String catName = database.getCatalogName();
String dbName = database.getName().toLowerCase();
boolean isDefaultCatalog = Warehouse.DEFAULT_CATALOG_NAME.equals(catName);

// external warehouse root
String whLocation = MetastoreConf.getVar(conf,
isDefaultCatalog ? MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL : MetastoreConf.ConfVars.WAREHOUSE_CATALOG_EXTERNAL,
MetastoreConf.getVar(conf,
isDefaultCatalog ? MetastoreConf.ConfVars.WAREHOUSE : MetastoreConf.ConfVars.WAREHOUSE_CATALOG));

if (!isDefaultCatalog) {
whLocation = new Path(whLocation, catName).toString();
}

Path dbDerivedLoc = new Path(whLocation, dbName + DATABASE_PATH_SUFFIX);
String defaultDbLoc = Utilities.getQualifiedPath((HiveConf) conf, dbDerivedLoc);
database.putToParameters(ReplConst.REPL_IS_CUSTOM_DB_LOC,
Boolean.toString(!defaultDbLoc.equals(database.getLocationUri())));
String whManagedLocatoion = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.WAREHOUSE);
Path dbDerivedManagedLoc = new Path(whManagedLocatoion, database.getName().toLowerCase()

// managed warehouse root
String whManagedLocatoion = MetastoreConf.getVar(conf,
isDefaultCatalog ? MetastoreConf.ConfVars.WAREHOUSE
: MetastoreConf.ConfVars.WAREHOUSE_CATALOG);

if (!isDefaultCatalog) {
whManagedLocatoion = new Path(whManagedLocatoion, catName).toString();
}
Path dbDerivedManagedLoc = new Path(whManagedLocatoion, dbName
+ DATABASE_PATH_SUFFIX);
String defaultDbManagedLoc = Utilities.getQualifiedPath((HiveConf) conf, dbDerivedManagedLoc);
database.getParameters().put(ReplConst.REPL_IS_CUSTOM_DB_MANAGEDLOC, Boolean.toString(
Expand Down
13 changes: 10 additions & 3 deletions ql/src/java/org/apache/hadoop/hive/ql/queryhistory/repository/AbstractRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.PartitionTransform;
import org.apache.hadoop.hive.ql.parse.TransformSpec;
Expand All @@ -40,6 +41,8 @@
import java.util.ArrayList;
import java.util.List;

import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;

public abstract class AbstractRepository implements QueryHistoryRepository {
protected Logger LOG = LoggerFactory.getLogger(getClass());
@VisibleForTesting
Expand Down Expand Up @@ -82,7 +85,11 @@ protected Database initDatabase(Hive hive) {
db = hive.getDatabase(QUERY_HISTORY_DB_NAME);
if (db == null) {
LOG.warn("Database ({}) for query history table hasn't been found, auto-creating one", QUERY_HISTORY_DB_NAME);
String location = getDatabaseLocation(QUERY_HISTORY_DB_NAME);
db = new Database();
// TODO catalog. The Hive Query History functionality is currently limited to the default catalog.
db.setCatalogName(getDefaultCatalog(conf));
db.setName(QUERY_HISTORY_DB_NAME);
String location = getDatabaseLocation(db);
db = new Database(QUERY_HISTORY_DB_NAME, QUERY_HISTORY_DB_COMMENT,
location, null);
hive.createDatabase(db, false);
Expand All @@ -93,8 +100,8 @@ protected Database initDatabase(Hive hive) {
}
}

private String getDatabaseLocation(String databaseName) throws Exception {
return warehouse.getDefaultExternalDatabasePath(databaseName).toUri().toString();
private String getDatabaseLocation(Database db) throws Exception {
return warehouse.getDefaultExternalDatabasePath(db).toUri().toString();
}

protected Table initTable(Hive hive, Database db) {
Expand Down
4 changes: 3 additions & 1 deletion ...a/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ public void authorizeDbLevelOperations(Privilege[] readRequiredPriv, Privilege[]
Path root = null;
try {
initWh();
// TODO catalog. Need to determine auth root path based on catalog name.
// Not sure how to get catalog name yet. Use getWhRoot(String) instead.
root = wh.getWhRoot();
// When we have some path in outputs, we should check access on that path, usually happens when
// we have HiveOperation.CREATEDATABASE query with some location
Expand Down Expand Up @@ -455,7 +457,7 @@ protected Path getDbLocation(Database db) throws HiveException {
initWh();
String location = db.getLocationUri();
if (location == null) {
return wh.getDefaultDatabasePath(db.getName());
return wh.getDefaultDatabasePath(db);
} else {
return wh.getDnsPath(wh.getDatabasePath(db));
}
Expand Down
Loading