diff --git a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql index aeab69311..80c672fee 100644 --- a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql @@ -348,6 +348,7 @@ VALUES ('agent', 'x86_64,aarch64', 'http://your-repo/', 'bigtop-manager-agent.tar.gz', null, 2), ('jdk8', 'x86_64', 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u452-b09/', 'OpenJDK8U-jdk_x64_linux_hotspot_8u452b09.tar.gz', 'SHA-256:9448308a21841960a591b47927cf2d44fdc4c0533a5f8111a4b243a6bafb5d27', 2), ('jdk8', 'aarch64', 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u452-b09/', 'OpenJDK8U-jdk_aarch64_linux_hotspot_8u452b09.tar.gz', 'SHA-256:d8a1aecea0913b7a1e0d737ba6f7ea99059b3f6fd17813d4a24e8b3fc3aee278', 2), +('jdk25', 'x86_64', 'https://github.com/adoptium/temurin25-binaries/releases/download/jdk-25.0.2%2B10/', 'OpenJDK25U-jdk_x64_linux_hotspot_25.0.2_10.tar.gz', 'sha256:987387933b64b9833846dee373b640440d3e1fd48a04804ec01a6dbf718e8ab8', 2), ('mysql-connector-j', 'x86_64,aarch64', 'https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/', 'mysql-connector-j-8.0.33.jar', 'SHA-256:e2a3b2fc726a1ac64e998585db86b30fa8bf3f706195b78bb77c5f99bf877bd9', 2); diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index 35feb63ff..4ef1c8665 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -361,6 +361,7 @@ VALUES ('agent', 'x86_64,aarch64', 'http://your-repo/', 'bigtop-manager-agent.tar.gz', null, 2), ('jdk8', 'x86_64', 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u452-b09/', 'OpenJDK8U-jdk_x64_linux_hotspot_8u452b09.tar.gz', 'SHA-256:9448308a21841960a591b47927cf2d44fdc4c0533a5f8111a4b243a6bafb5d27', 2), ('jdk8', 'aarch64', 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u452-b09/', 'OpenJDK8U-jdk_aarch64_linux_hotspot_8u452b09.tar.gz', 'SHA-256:d8a1aecea0913b7a1e0d737ba6f7ea99059b3f6fd17813d4a24e8b3fc3aee278', 2), +('jdk25', 'x86_64', 'https://github.com/adoptium/temurin25-binaries/releases/download/jdk-25.0.2%2B10/', 'OpenJDK25U-jdk_x64_linux_hotspot_25.0.2_10.tar.gz', 'sha256:987387933b64b9833846dee373b640440d3e1fd48a04804ec01a6dbf718e8ab8', 2), ('mysql-connector-j', 'x86_64,aarch64', 'https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/', 'mysql-connector-j-8.0.33.jar', 'SHA-256:e2a3b2fc726a1ac64e998585db86b30fa8bf3f706195b78bb77c5f99bf877bd9', 2); INSERT INTO llm_platform (credential, name, support_models) diff --git a/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/config.properties.xml b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/config.properties.xml new file mode 100644 index 000000000..ca59bb75b --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/config.properties.xml @@ -0,0 +1,38 @@ + + + + + + http-server.http.port + + Specifies the port for the HTTP server. Trino uses HTTP for all communication, internal and external. + + 8080 + + + node-scheduler.include-coordinator + + Allow scheduling work on the coordinator. For larger clusters, processing work on the coordinator + can impact query performance because the machine’s resources are not available for the critical task of + scheduling, managing and monitoring query execution. + + false + + diff --git a/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/hive.properties.xml b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/hive.properties.xml new file mode 100644 index 000000000..e099c95fb --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/hive.properties.xml @@ -0,0 +1,43 @@ + + + + + + hive.storage-format + + The default file format used when creating new tables. + + ORC + + + hive.translate-hive-views + + hive translate hive views. + + true + + + fs.hadoop.enabled + + Activate the support for HDFS access. Defaults to false. Set to true to use HDFS and enable all other properties. + + true + + diff --git a/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/jvm.config.xml b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/jvm.config.xml new file mode 100644 index 000000000..8de95e6d2 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/jvm.config.xml @@ -0,0 +1,67 @@ + + + + + + content + This is the freemarker template for jvm.config file + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +-server +-Xmx2G +-XX:InitialRAMPercentage=80 +-XX:MaxRAMPercentage=80 +-XX:G1HeapRegionSize=32M +-XX:+ExplicitGCInvokesConcurrent +-XX:+ExitOnOutOfMemoryError +-XX:+HeapDumpOnOutOfMemoryError +-XX:-OmitStackTraceInFastThrow +-XX:ReservedCodeCacheSize=512M +-XX:PerMethodRecompilationCutoff=10000 +-XX:PerBytecodeRecompilationCutoff=10000 +-Djdk.attach.allowAttachSelf=true +-Djdk.nio.maxCachedBufferSize=2000000 +-Dfile.encoding=UTF-8 +# Allow loading dynamic agent used by JOL +-XX:+EnableDynamicAgentLoading + +]]> + + + longtext + + + diff --git a/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/node.properties.xml b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/node.properties.xml new file mode 100644 index 000000000..c28e6ce5f --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/configuration/node.properties.xml @@ -0,0 +1,38 @@ + + + + + + node.environment + + The name of the environment. All Trino nodes in a cluster must have the same environment name. The name must + start with a lowercase alphanumeric character and only contain lowercase alphanumeric or underscore (_) + characters. + + production + + + node.data-dir + + The location (filesystem path) of the data directory. Trino stores logs and other data here. + + /var/trino/data + + diff --git a/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/metainfo.xml new file mode 100644 index 000000000..ecc9b98dd --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/metainfo.xml @@ -0,0 +1,69 @@ + + + + + + trino + Trino + + Trino is a distributed SQL query engine designed to query large data sets distributed over one or more + heterogeneous data sources. + + 479 + trino + Apache-2.0 + + + + trino_worker + Trino Worker + server + 0+ + + + trino_coordinator + Trino Coordinator + server + 1 + + + trino_client + Trino Client + client + 0+ + + + + + + + x86_64 + aarch64 + + + + trino-server-479.tar.gz + SHA-256:ccc667a23400eb894bbb1e58d441ef54b04fde50f3e6a7723af84696a3bc5f92 + + + + + + diff --git a/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/order.json b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/order.json new file mode 100644 index 000000000..0db3279e4 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/extra/1.0.0/services/trino/order.json @@ -0,0 +1,3 @@ +{ + +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java index af2da9f4b..802459778 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java @@ -93,7 +93,13 @@ private static boolean extract(TarArchiveInputStream ais, Path destDir, int skip if (entry.isDirectory()) { createDirectories(outputPath); } else if (entry.isSymbolicLink()) { - createSymbolicLink(outputPath, entry.getLinkName()); + Path linkNamePath = Paths.get(entry.getLinkName()); + Path targetLink = linkNamePath.subpath(skipLevels, linkNamePath.getNameCount()); + createSymbolicLink(outputPath, destDir.resolve(targetLink).normalize()); + } else if (entry.isLink()) { + Path linkNamePath = Paths.get(entry.getLinkName()); + Path targetLink = linkNamePath.subpath(skipLevels, linkNamePath.getNameCount()); + createLink(outputPath, destDir.resolve(targetLink).normalize()); } else { createFile(outputPath, ais); } @@ -116,18 +122,24 @@ private static void createDirectories(Path path) { } } - private static void createSymbolicLink(Path linkPath, String targetName) { - Path targetPath = linkPath.getParent().resolve(targetName).normalize(); - if (!targetPath.isAbsolute()) { - targetPath = linkPath.getParent().resolve(targetPath).normalize(); + private static void createSymbolicLink(Path linkPath, Path targetLink) { + createDirectories(linkPath.getParent()); + + try { + Files.createSymbolicLink(linkPath, targetLink); + } catch (IOException e) { + log.error("Failed to create symbolic link from {} to {}", new Object[] {linkPath, targetLink, e}); + throw new StackException(e); } + } + private static void createLink(Path linkPath, Path targetLink) { createDirectories(linkPath.getParent()); try { - Files.createSymbolicLink(linkPath, targetPath); + Files.createLink(linkPath, targetLink); } catch (IOException e) { - log.error("Failed to create symbolic link from {} to {}", linkPath, targetPath, e); + log.error("Failed to create link from {} to {}", new Object[] {linkPath, targetLink, e}); throw new StackException(e); } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/AbstractTrinoScript.java b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/AbstractTrinoScript.java new file mode 100644 index 000000000..bd0978ee3 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/AbstractTrinoScript.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.extra.v1_0_0.trino; + +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.grpc.pojo.ClusterInfo; +import org.apache.bigtop.manager.grpc.pojo.PackageInfo; +import org.apache.bigtop.manager.grpc.pojo.RepoInfo; +import org.apache.bigtop.manager.stack.core.exception.StackException; +import org.apache.bigtop.manager.stack.core.spi.param.Params; +import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; +import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import org.apache.bigtop.manager.stack.core.utils.TarballUtils; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; + +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.Properties; + +@Slf4j +public abstract class AbstractTrinoScript extends AbstractServerScript { + + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + installJdk((TrinoParams) params); + return super.add(params, properties); + } + + private void installJdk(TrinoParams params) { + log.info("Setting up trino jdk25..."); + ClusterInfo clusterInfo = LocalSettings.cluster(); + String dependenciesHome = clusterInfo.getRootDir() + "/dependencies"; + String user = System.getProperty("user.name"); + LinuxFileUtils.createDirectories(dependenciesHome, user, user, Constants.PERMISSION_755, true); + + String jdkHome = dependenciesHome + "/jdk25"; + RepoInfo repoInfo = LocalSettings.repo("jdk25"); + PackageInfo packageInfo = new PackageInfo(); + packageInfo.setName(repoInfo.getPkgName()); + packageInfo.setChecksum(repoInfo.getChecksum()); + TarballUtils.installPackage(repoInfo.getBaseUrl(), dependenciesHome, jdkHome, packageInfo, 1); + LinuxFileUtils.createDirectories(jdkHome, user, user, Constants.PERMISSION_755, true); + } + + @Override + public ShellResult configure(Params params) { + super.configure(params); + return TrinoSetup.config((TrinoParams) params, getComponentName()); + } + + @Override + public ShellResult start(Params params) { + TrinoParams trinoParams = (TrinoParams) params; + + String cmd = MessageFormat.format( + "JAVA_HOME={0} {1}/bin/launcher start", trinoParams.javaHome(), trinoParams.serviceHome()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, trinoParams.user()); + } catch (Exception e) { + throw new StackException(e); + } + } + + @Override + public ShellResult stop(Params params) { + TrinoParams trinoParams = (TrinoParams) params; + + String cmd = MessageFormat.format( + "JAVA_HOME={0} {1}/bin/launcher stop", trinoParams.javaHome(), trinoParams.serviceHome()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, trinoParams.user()); + } catch (Exception e) { + throw new StackException(e); + } + } + + @Override + public ShellResult status(Params params) { + TrinoParams trinoParams = (TrinoParams) params; + + String cmd = MessageFormat.format( + "JAVA_HOME={0} {1}/bin/launcher status", trinoParams.javaHome(), trinoParams.serviceHome()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, trinoParams.user()); + } catch (Exception e) { + throw new StackException(e); + } + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoClientScript.java new file mode 100644 index 000000000..c583a6daa --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoClientScript.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.extra.v1_0_0.trino; + +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.grpc.pojo.RepoInfo; +import org.apache.bigtop.manager.stack.core.spi.param.Params; +import org.apache.bigtop.manager.stack.core.spi.script.Script; +import org.apache.bigtop.manager.stack.core.tarball.FileDownloader; +import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; + +@Slf4j +@AutoService(Script.class) +public class TrinoClientScript extends AbstractTrinoScript { + + @Override + public ShellResult add(Params params) { + String cliFileName = "trino-cli-479"; + String downTmp = "/tmp"; + String binPath = "/usr/bin/"; + RepoInfo generalRepo = LocalSettings.repo("general"); + FileDownloader.download(getCliDownloadPath(generalRepo.getBaseUrl(), cliFileName), downTmp); + LinuxFileUtils.moveFile(downTmp + File.separator + cliFileName, binPath + cliFileName); + LinuxFileUtils.updateOwner(binPath + cliFileName, params.user(), params.group(), false); + LinuxFileUtils.updatePermissions(binPath + cliFileName, Constants.PERMISSION_755, false); + return ShellResult.success(); + } + + @Override + public ShellResult configure(Params params) { + return super.configure(params); + } + + @Override + public String getComponentName() { + return "trino_client"; + } + + private String getCliDownloadPath(String basePath, String fileName) { + if (basePath.endsWith(File.separator) && fileName.startsWith(File.separator)) { + return basePath + fileName.substring(1); + } else if (basePath.endsWith(File.separator) || fileName.startsWith(File.separator)) { + return basePath + fileName; + } else { + return basePath + File.separator + fileName; + } + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoCoordinatorScript.java b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoCoordinatorScript.java new file mode 100644 index 000000000..d794de9c0 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoCoordinatorScript.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.extra.v1_0_0.trino; + +import org.apache.bigtop.manager.stack.core.spi.script.Script; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@AutoService(Script.class) +public class TrinoCoordinatorScript extends AbstractTrinoScript { + + @Override + public String getComponentName() { + return "trino_coordinator"; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoParams.java b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoParams.java new file mode 100644 index 000000000..6e1bd4f14 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoParams.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.extra.v1_0_0.trino; + +import org.apache.bigtop.manager.grpc.payload.ComponentCommandPayload; +import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.spi.param.Params; +import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import org.apache.bigtop.manager.stack.extra.param.ExtraParams; + +import com.google.auto.service.AutoService; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.List; +import java.util.Map; + +@Slf4j +@Getter +@NoArgsConstructor +@AutoService(Params.class) +public class TrinoParams extends ExtraParams { + + private String jvmConfigContent; + + public TrinoParams(ComponentCommandPayload payload) { + super(payload); + globalParamsMap.put("java_home", javaHome()); + globalParamsMap.put("trino_user", user()); + globalParamsMap.put("trino_group", group()); + globalParamsMap.put("trino_home", serviceHome()); + globalParamsMap.put("trino_conf_dir", confDir()); + } + + @Override + public String getServiceName() { + return "trino"; + } + + @Override + public String javaHome() { + String root = LocalSettings.cluster().getRootDir(); + return MessageFormat.format("{0}/dependencies/jdk25", root); + } + + @Override + public String confDir() { + return serviceHome() + "/etc"; + } + + public List coordinatorHosts() { + return LocalSettings.componentHosts("coordinator"); + } + + @GlobalParams + public Map nodeProperties() { + return LocalSettings.configurations(getServiceName(), "node.properties"); + } + + @GlobalParams + public Map jvmConfig() { + Map configuration = LocalSettings.configurations(getServiceName(), "jvm.config"); + jvmConfigContent = configuration.get("content").toString(); + return configuration; + } + + @GlobalParams + public Map configProperties() { + Map configuration = LocalSettings.configurations(getServiceName(), "config.properties"); + Object port = configuration.get("http-server.http.port"); + List hosts = coordinatorHosts(); + if (!hosts.isEmpty()) { + configuration.put("discovery.uri", MessageFormat.format("http://{0}:{1}", hosts.get(0), port)); + } + return configuration; + } + + @GlobalParams + public Map hiveProperties() { + Map configuration = LocalSettings.configurations(getServiceName(), "hive.properties"); + return configuration; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoSetup.java b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoSetup.java new file mode 100644 index 000000000..c2a68dfaf --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoSetup.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.extra.v1_0_0.trino; + +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.enums.ConfigType; +import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; + +import org.apache.commons.lang3.StringUtils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class TrinoSetup { + + public static ShellResult config(TrinoParams params, String componentName) { + + String user = params.user(); + String group = params.group(); + + Map globalParamsMap = params.getGlobalParamsMap(); + + LinuxFileUtils.createDirectories(params.confDir(), user, group, Constants.PERMISSION_755, true); + + // create data dir + Map nodeProperties = params.nodeProperties(); + Object dataDir = nodeProperties.get("node.data-dir"); + if (dataDir != null && StringUtils.isNotBlank(dataDir.toString())) { + LinuxFileUtils.createDirectories(dataDir.toString(), user, group, Constants.PERMISSION_755, true); + } + + Map nodeMap = new HashMap<>(nodeProperties); + + nodeMap.put("node.id", params.hostname()); + + Map configProperties = new HashMap<>(params.configProperties()); + configProperties.put("coordinator", StringUtils.equalsIgnoreCase(componentName, "trino_coordinator")); + + LinuxFileUtils.toFile( + ConfigType.PROPERTIES, + MessageFormat.format("{0}/node.properties", params.confDir()), + user, + group, + Constants.PERMISSION_644, + nodeMap); + + LinuxFileUtils.toFileByTemplate( + params.getJvmConfigContent(), + MessageFormat.format("{0}/jvm.config", params.confDir()), + user, + group, + Constants.PERMISSION_644, + globalParamsMap); + + LinuxFileUtils.toFile( + ConfigType.PROPERTIES, + MessageFormat.format("{0}/config.properties", params.confDir()), + user, + group, + Constants.PERMISSION_644, + configProperties); + + // create catalog dir and catalog.properties + LinuxFileUtils.createDirectories( + MessageFormat.format("{0}/catalog", params.confDir()), user, group, Constants.PERMISSION_755, true); + + Map hiveCatalog = createHiveCatalog(params); + if (!hiveCatalog.isEmpty()) { + LinuxFileUtils.toFile( + ConfigType.PROPERTIES, + MessageFormat.format("{0}/catalog/hive.properties", params.confDir()), + user, + group, + Constants.PERMISSION_644, + hiveCatalog); + } + + return ShellResult.success("Trino Configure success!"); + } + + private static Map createHiveCatalog(TrinoParams params) { + Map hiveCatalog = params.hiveProperties(); + hiveCatalog.put("connector.name", "hive"); + + Map hiveSite = LocalSettings.configurations("hive", "hive-site"); + String metastoreUris = hiveSite.get("hive.metastore.uris").toString(); + String[] split = metastoreUris.split(":"); + String metastorePort = split[split.length - 1]; + + List hiveMetastores = LocalSettings.componentHosts("hive_metastore"); + if (hiveMetastores != null && !hiveMetastores.isEmpty()) { + String hiveMetastore = hiveMetastores.get(0); + hiveCatalog.put( + "hive.metastore.uri", MessageFormat.format("thrift://{0}:{1}", hiveMetastore, metastorePort)); + } else { + return new HashMap<>(); + } + return hiveCatalog; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoWorkerScript.java b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoWorkerScript.java new file mode 100644 index 000000000..082ef8471 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/trino/TrinoWorkerScript.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.extra.v1_0_0.trino; + +import org.apache.bigtop.manager.stack.core.spi.script.Script; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@AutoService(Script.class) +public class TrinoWorkerScript extends AbstractTrinoScript { + + @Override + public String getComponentName() { + return "trino_worker"; + } +} diff --git a/bigtop-manager-ui/src/assets/images/trino.png b/bigtop-manager-ui/src/assets/images/trino.png new file mode 100644 index 000000000..85d34cb8b Binary files /dev/null and b/bigtop-manager-ui/src/assets/images/trino.png differ