package com.taobao.hsf.tlog.config.shell; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; abstract class Shell { public static class ShellCommandExecutor extends Shell { private final String[] command; private StringBuffer output; public ShellCommandExecutor(String[] execString) { command = execString.clone(); } public ShellCommandExecutor(String[] execString, File dir) { this(execString); this.setWorkingDirectory(dir); } public ShellCommandExecutor(String[] execString, File dir, Map<String, String> env) { this(execString, dir); this.setEnvironment(env); } public void execute() throws IOException { this.run(); } public String getOutput() { return (output == null) ? "" : output.toString(); } @Override protected String[] getExecString() { return command; } @Override protected void parseExecResult(BufferedReader lines) throws IOException { output = new StringBuffer(); char[] buf = new char[512]; int nRead; while ((nRead = lines.read(buf, 0, buf.length)) > 0) { output.append(buf, 0, nRead); } } } static class ExitCodeException extends IOException { private static final long serialVersionUID = -7102633161419484030L; int exitCode; public ExitCodeException(int exitCode, String message) { super(message); this.exitCode = exitCode; } public int getExitCode() { return exitCode; } } private static final Logger logger = LoggerFactory.getLogger(Shell.class); public static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows"); public static String execCommand(Map<String, String> env, String... cmd) throws IOException { ShellCommandExecutor exec = new ShellCommandExecutor(cmd); if (env != null) { exec.setEnvironment(env); } exec.execute(); return exec.getOutput(); } public static String execCommand(String... cmd) throws IOException { return execCommand(null, cmd); } private Map<String, String> environment; private File dir; private Process process; private int exitCode; public int getExitCode() { return exitCode; } public Process getProcess() { return process; } protected abstract String[] getExecString(); protected abstract void parseExecResult(BufferedReader lines) throws IOException; protected void run() throws IOException { exitCode = 0; //reset runCommand(); } protected void setEnvironment(Map<String, String> env) { this.environment = env; } protected void setWorkingDirectory(File dir) { this.dir = dir; } private void runCommand() throws IOException { ProcessBuilder builder = new ProcessBuilder(getExecString()); boolean completed = false; if (environment != null) { builder.environment().putAll(this.environment); } if (dir != null) { builder.directory(this.dir); } process = builder.start(); final BufferedReader errReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); final StringBuffer errMsg = new StringBuffer(); Thread errThread = new Thread() { @Override public void run() { try { String line = errReader.readLine(); while ((line != null) && !isInterrupted()) { errMsg.append(line); errMsg.append(System.getProperty("line.separator")); line = errReader.readLine(); } } catch (IOException ioe) { logger.warn("Error reading the error stream", ioe); } } }; try { errThread.start(); } catch (IllegalStateException ise) { } try { parseExecResult(inReader); exitCode = process.waitFor(); try { errThread.join(); } catch (InterruptedException ie) { logger.warn("Interrupted while reading the error stream", ie); } completed = true; if (exitCode != 0) { throw new ExitCodeException(exitCode, errMsg.toString()); } } catch (InterruptedException ie) { throw new IOException(ie.toString()); } finally { try { inReader.close(); } catch (IOException ioe) { logger.warn("Error while closing the input stream", ioe); } if (!completed) { errThread.interrupt(); } try { errReader.close(); } catch (IOException ioe) { logger.warn("Error while closing the error stream", ioe); } process.destroy(); } } } /* * 使用Shell封装 */ public class StormScheduler { private final static File workDir = new File("/home/admin"); public static void main(String[] args) { schedule("tlog"); } public static void schedule(String appName) { System.out.println("Start to execute schedule shell: '/bin/bash storm_scheduler.sh " + appName + "'"); Shell.ShellCommandExecutor executor = new Shell.ShellCommandExecutor(new String[] { "/bin/bash", "storm_scheduler.sh", appName }, workDir); try { executor.execute(); } catch (IOException e) { System.out.println("Shell Exception, " + e); throw new RuntimeException("StormScheduler error, appName= " + appName, e); } System.out.println("Shell output: " + executor.getOutput()); } }
相关推荐
hadoop 部分 shell 脚本
适用于《shell脚本配置伪分布式Hadoop》
hadoop集群关机重启shell脚本
Hadoop高可用自动化安装使用Shell脚本,可自行修改适配自己的集群,完全自动化,仅提供脚本,软件自备
利用shell脚本 对Hadoop环境进行傻瓜式配置 先解压! 环节包括: 1.修改hostname 2.修改hosts 3.配置免密 4.Hadoop配置文件的更改 !!!!!!!!!!!!!!!!!!!! ps 请特别注意以下几个问题: 1....
docx
#shell脚本完成hadoop的集群安装 #步骤: #1.安装虚拟机,关闭防火墙、selinux #2.ssh免密码,编辑hosts文件 #3.安装JDK #4.安装hadoop #5.修改配置文件 #6.分发hadoop程序到各个节点 #7.启动集群
在集群上设置 Hadoop 的 Shell 脚本和说明。 旨在与 Amazon EC2 上的 Ubuntu Server 14.04.1 LTS 的新实例一起使用。 下面的说明显示了如何设置 2 节点集群。 创建实例 首先,使用 Amazon Web Interface 创建两个...
为hive-0.11半自动安装脚本 使用前请先阅读本脚本注释部分 已有hadoop环境可使用本脚本 因为初识shell脚本 望大虾勿喷 如有不吝赐教者 不胜感激
安装hadoop时,集群式安装需要来回的切换机器,那么还有每一步都需要配置文件,很繁琐,所以就在这写了一个脚本
大数据集群管理脚本
介绍了Hadoop bin下所有脚本以及Hadoop Shell使用介绍
这个shell脚本是在业余时间写的,从一开始不懂shell,到写出这个程序还是经过了一段时间的,收取小小1分希望得到大家的鼓励 :) 。程序的解释和hadoop多用户配置的步骤也都可以在博客中找到:)
一键安装Hadoop集群环境shell脚本,默认安装Hadoop-3.2.1版本,安装其他版本更改Hadoop安装包链接即可
感谢我关注的朋友,我亦无他,收集整理的大部分脚本。 群发脚本XSync(xsync) #!/bin/bash #1 获取参数,如果不够直接退出 pcount=$# if ((pcount==0)); then echo 没有输入需要传递的文件或文件夹!!; exit; fi #2 ...
需要自己先安装jdk,也可以去这里https://download.csdn.net/download/weixin_44033210/12657718,这里有自动安装jdk的脚本
需要自己先安装jdk和Hadoop,也可以去这里https://download.csdn.net/download/weixin_44033210/12657718,https://download.csdn.net/download/weixin_44033210/12657738,这里有自动安装jdk和Hadoop的脚本
CentOS 7 下的 Hadoop 一键脚本使用方法脚本完成的是Hadoop分布式集群配置包含wordcount例子 (在wordcount文件夹下面)单机版配置方法请看下面的注意事项前提条件:登录用户具有sudo权限(root也可)具有网络连接安装...
hadoop集群管理脚本
一件分发脚本,在使用过程中,更改服务器名称即可,一键搭建Hadoop不是梦,直接配置一个Hadoop,用该脚本分发,瞬间完成服务器搭建,为开发人员节约了大量的时间,和精力。