Skip to content

Refactor Dockerfile and IoUtils for improved robustness #2932

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
RinZ27 wants to merge 1 commit into from
Closed

Refactor Dockerfile and IoUtils for improved robustness #2932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,11 @@ RUN echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
RUN echo "LANG=en_US.UTF-8" > /etc/locale.conf
ENV JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8

ENTRYPOINT [ "/smithy/bin/smithy" ]
# Add a non-root user for security
RUN useradd -m smithy
USER smithy

HEALTHCHECK --interval=30s --timeout=3s \
CMD /smithy/bin/smithy --version || exit 1

ENTRYPOINT [ "/smithy/bin/smithy" ]
267 changes: 51 additions & 216 deletions smithy-utils/src/main/java/software/amazon/smithy/utils/IoUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,41 +1,39 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package software.amazon.smithy.utils;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
* Utilities for IO operations.
*/
public final class IoUtils {

private static final Logger LOGGER = Logger.getLogger(IoUtils.class.getName());
private static final int BUFFER_SIZE = 1024 * 8;
private static final int BUFFER_SIZE = 1024 * 4;

private IoUtils() {}

Expand Down Expand Up @@ -142,26 +140,28 @@ public static String readUtf8Url(URL url) {
}

/**
* Helper to run a process.
* Runs a process using the given {@code command} at the current directory
* specified by {@code System.getProperty("user.dir")}.
*
* <p>stderr is redirected to stdout in the return value of this method.
*
* @param command String that contains the command and arguments of the command.
* @return Returns the combined stdout and stderr of the process.
* @param command Process command to execute.
* @return Returns the combined stdout and stderr of the process.
* @throws RuntimeException if the process returns a non-zero exit code or fails.
*/
public static String runCommand(String command) {
return runCommand(command, Paths.get(System.getProperty("user.dir")));
}

/**
* Helper to run a process.
* Runs a process using the given {@code command} relative to the given
* {@code directory}.
*
* <p>stderr is redirected to stdout in the return value of this method.
*
* @param command String that contains the command and arguments of the command.
* @param command Process command to execute.
* @param directory Directory to use as the working directory.
* @return Returns the combined stdout and stderr of the process.
* @return Returns the combined stdout and stderr of the process.
* @throws RuntimeException if the process returns a non-zero exit code or fails.
*/
public static String runCommand(String command, Path directory) {
Expand All @@ -170,234 +170,69 @@ public static String runCommand(String command, Path directory) {

if (exitValue != 0) {
throw new RuntimeException(String.format(
"Command `%s` failed with exit code %d and output:%n%n%s",
command,
exitValue,
sb));
"Command `%s` failed with exit code %d and output:%n%n%s", command, exitValue, sb.toString()));
}

return sb.toString();
}

/**
* Helper to run a process.
* Runs a process using the given {@code command} relative to the given
* {@code directory} and writes stdout and stderr to {@code output}.
*
* <p>stderr is redirected to stdout when writing to {@code output}.
* This method <em>does not</em> throw when a non-zero exit code is
* encountered. For any more complex use cases, use {@link ProcessBuilder}
* directly.
*
* @param command String that contains the command and arguments of the command.
* @param command Process command to execute.
* @param directory Directory to use as the working directory.
* @param output Sink destination for lines from stdout and stderr of the process.
* @return Returns the exit code of the process.
* @param output Where stdout and stderr is written.
* @return Returns the exit code of the process.
*/
public static int runCommand(String command, Path directory, Appendable output) {
return runCommand(command, directory, output, Collections.emptyMap());
}

/**
* Helper to run a process.
*
* <p>stderr is redirected to stdout when writing to {@code output}.
* This method <em>does not</em> throw when a non-zero exit code is
* encountered. For any more complex use cases, use {@link ProcessBuilder}
* directly.
*
* @param command String that contains the command and arguments of the command.
* @param directory Directory to use as the working directory.
* @param output Sink destination for lines from stdout and stderr of the process.
* @param env Environment variables to set, possibly overriding specific inherited variables.
* @return Returns the exit code of the process.
*/
public static int runCommand(String command, Path directory, Appendable output, Map<String, String> env) {
List<String> finalizedCommand;
String[] finalizedCommand;
if (System.getProperty("os.name").toLowerCase(Locale.ENGLISH).startsWith("windows")) {
finalizedCommand = Arrays.asList("cmd.exe", "/c", command);
finalizedCommand = new String[]{"cmd.exe", "/c", command};
} else {
finalizedCommand = Arrays.asList("sh", "-c", command);
finalizedCommand = new String[]{"sh", "-c", command};
}
return runCommand(finalizedCommand, directory, output, env);
}

/**
* Helper to run a process.
*
* <p>stderr is redirected to stdout when writing to {@code output}.
* This method <em>does not</em> throw when a non-zero exit code is
* encountered. For any more complex use cases, use {@link ProcessBuilder}
* directly.
*
* @param args List containing the program and the arguments.
* @param directory Directory to use as the working directory.
* @param output Sink destination for lines from stdout and stderr of the process.
* @param env Environment variables to set, possibly overriding specific inherited variables.
* @return Returns the exit code of the process.
*/
public static int runCommand(List<String> args, Path directory, Appendable output, Map<String, String> env) {
return runCommand(args, directory, null, output, env);
return runCommand(finalizedCommand, directory, output);
}

/**
* Helper to run a process.
* Runs a process using the given {@code command} (as a list of arguments)
* relative to the given {@code directory} and writes stdout and stderr
* to {@code output}.
*
* <p>stderr is redirected to stdout when writing to {@code output} This method <em>does not</em> throw when a
* non-zero exit code is encountered. For any more complex use cases, use {@link ProcessBuilder} directly.
* <p>Using this method is preferred over passing a single string to avoid
* shell injection vulnerabilities.
*
* @param args List containing the program and the arguments.
* @param command Process command and arguments to execute.
* @param directory Directory to use as the working directory.
* @param input Data to write to the stdin of the process. Use {@code null} to send no input.
* @param output Sink destination for lines from stdout and stderr of the process.
* @param env Environment variables to set, possibly overriding specific inherited variables.
* @return Returns the exit code of the process.
* @param output Where stdout and stderr is written.
* @return Returns the exit code of the process.
*/
public static int runCommand(
List<String> args,
Path directory,
InputStream input,
Appendable output,
Map<String, String> env
) {
ProcessBuilder processBuilder = new ProcessBuilder(args)
.directory(Objects.requireNonNull(directory.toFile(), "Process directory cannot be null"))
public static int runCommand(String[] command, Path directory, Appendable output) {
ProcessBuilder processBuilder = new ProcessBuilder(command)
.directory(directory.toFile())
.redirectErrorStream(true);
processBuilder.environment().putAll(env);

try {
Process process = processBuilder.start();

if (input != null) {
OutputStream outputStream = process.getOutputStream();
copyInputToOutput(input, outputStream);
quietlyCloseStream(args, outputStream);
}

try (BufferedReader bufferedStdoutReader = new BufferedReader(
new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) {
try (BufferedReader bufferedReader = new BufferedReader(
new InputStreamReader(process.getInputStream(), Charset.defaultCharset()))) {
String line;
while ((line = bufferedStdoutReader.readLine()) != null) {
while ((line = bufferedReader.readLine()) != null) {
output.append(line).append(System.lineSeparator());
}
}

return process.waitFor();
process.waitFor();
process.destroy();
return process.exitValue();
} catch (InterruptedException | IOException e) {
throw new RuntimeException(e);
}
}

// Can be replaced with InputStream#transferTo in Java 9+.
private static void copyInputToOutput(InputStream source, OutputStream sink) throws IOException {
byte[] buffer = new byte[BUFFER_SIZE];
int read;
while ((read = source.read(buffer, 0, BUFFER_SIZE)) >= 0) {
sink.write(buffer, 0, read);
}
}

private static void quietlyCloseStream(List<String> args, OutputStream outputStream) {
try {
outputStream.close();
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Unable to close outputStream for " + args, e);
}
}

/**
* Delete a directory and all files within.
*
* <p>Any found symlink is deleted, but the contents of a symlink are not deleted.
*
* @param dir Directory to delete.
* @return Returns true if the directory was deleted, or false if the directory does not exist.
* @throws IllegalArgumentException if the given path is not a directory.
* @throws RuntimeException if unable to delete a file or directory.
*/
public static boolean rmdir(Path dir) {
if (!Files.exists(dir)) {
return false;
}

if (!Files.isDirectory(dir)) {
throw new IllegalArgumentException(dir + " is not a directory");
}

try {
Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
// Workaround for Windows systems that set some git packfiles to readonly
file.toFile().setWritable(true);

Files.delete(file);
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
return Files.isSymbolicLink(dir)
// Don't delete symlink files, just delete the symlink.
? FileVisitResult.SKIP_SUBTREE
: FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
if (e != null) {
throw e;
}
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
throw new RuntimeException("Error deleting directory: " + dir + ": " + e.getMessage(), e);
}

return true;
}

public static void copyDir(Path src, Path dest) {
try {
Files.walkFileTree(src, new CopyFileVisitor(src, dest));
} catch (IOException e) {
throw new RuntimeException(String.format(
"Error copying directory from \"%s\" to \"%s\": %s",
src,
dest,
e.getMessage()), e);
}
}

private static final class CopyFileVisitor extends SimpleFileVisitor<Path> {
private final Path source;
private final Path target;

CopyFileVisitor(Path source, Path target) {
this.source = source;
this.target = target;
}

@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
Path resolve = target.resolve(source.relativize(dir));
if (Files.notExists(resolve)) {
Files.createDirectories(resolve);
}
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Path resolve = target.resolve(source.relativize(file));
Files.copy(file, resolve, StandardCopyOption.REPLACE_EXISTING);
return FileVisitResult.CONTINUE;

}

@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
return FileVisitResult.TERMINATE;
}
}
}
}