JunRuiLee commented on code in PR #23606: URL: https://github.com/apache/flink/pull/23606#discussion_r1407834850
########## flink-core/src/main/java/org/apache/flink/configuration/YamlParserUtils.java: ########## @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.configuration; + +import org.apache.flink.util.TimeUtils; + +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.DumperOptions; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.LoaderOptions; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.Yaml; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.constructor.Constructor; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.error.Mark; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.error.MarkedYAMLException; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.error.YAMLException; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.nodes.Node; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.nodes.Tag; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.representer.Represent; +import org.apache.flink.shaded.jackson2.org.yaml.snakeyaml.representer.Representer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.time.Duration; +import java.util.Map; + +/** + * This class contains utility methods to load standard yaml file and convert object to standard + * yaml syntax. + */ +public class YamlParserUtils { + + private static final Logger LOG = LoggerFactory.getLogger(YamlParserUtils.class); + + private static final Yaml yaml; + + private static final DumperOptions dumperOptions = new DumperOptions(); + + private static final LoaderOptions loaderOptions = new LoaderOptions(); + + static { + // Make the dump output is in single line + dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.FLOW); + dumperOptions.setWidth(Integer.MAX_VALUE); + // The standard YAML do not allow duplicate keys. + loaderOptions.setAllowDuplicateKeys(false); + + yaml = + new Yaml( + new Constructor(loaderOptions), + new FlinkConfigRepresenter(dumperOptions), + dumperOptions, + loaderOptions); + } + + public static synchronized Map<String, Object> loadYamlFile(File file) throws Exception { + try (FileInputStream inputStream = new FileInputStream((file))) { + return yaml.load(inputStream); + } catch (FileNotFoundException e) { + LOG.error("Failed to find YAML file", e); + throw e; + } catch (IOException | YAMLException e) { + if (e instanceof MarkedYAMLException) { + YAMLException exception = + wrapExceptionToHiddenSensitiveData((MarkedYAMLException) e); + LOG.error("Failed to parse YAML configuration", exception); + throw exception; + } else { + throw e; + } + } + } + + /** + * Converts the given value to a string representation in the YAML syntax for configuration + * data. + * + * <p>NOTE: It will skip escaping string values in order to avoid ambiguity. + * + * @param value The value to be converted. + * @return The string representation of the value in YAML syntax. + */ + public static String toConfigurationDataString(Object value) { + if (value instanceof String) { + return value.toString(); + } else { + return toYAMLString(value); + } + } + + /** + * Converts the given value to a string representation in the YAML syntax. This method uses a + * YAML parser to convert the object to YAML format. + * + * <p>The resulting YAML string may have line breaks at the end of each line. This method + * removes the line break at the end of the string if it exists. + * + * <p>Note: This method may perform escaping on certain characters in the value to ensure proper + * YAML syntax. + * + * @param value The value to be converted. + * @return The string representation of the value in YAML syntax. + */ + public static synchronized String toYAMLString(Object value) { + try { + String output = yaml.dump(value); + // remove the line break + String linebreak = dumperOptions.getLineBreak().getString(); + if (output.endsWith(linebreak)) { + output = output.substring(0, output.length() - linebreak.length()); + } + return output; + } catch (MarkedYAMLException exception) { + throw wrapExceptionToHiddenSensitiveData(exception); + } + } + + public static synchronized <T> T convertToObject(String value, Class<T> type) { + try { + return yaml.loadAs(value, type); + } catch (MarkedYAMLException exception) { + throw wrapExceptionToHiddenSensitiveData(exception); + } + } + + /** + * This method is simply mocked the MarkedYAMLException#getMessage() to hidden sensitive data. + */ + private static YAMLException wrapExceptionToHiddenSensitiveData(MarkedYAMLException exception) { + StringBuilder lines = new StringBuilder(); + String context = exception.getContext(); + Mark contextMark = exception.getContextMark(); + String problem = exception.getProblem(); + Mark problemMark = exception.getProblemMark(); + + if (context != null) { + lines.append(context); + lines.append("\n"); + } + if (contextMark != null + && (problem == null + || problemMark == null + || contextMark.getName().equals(problemMark.getName()) + || (contextMark.getLine() != problemMark.getLine()) + || (contextMark.getColumn() != problemMark.getColumn()))) { + lines.append(hiddenSensitiveDataInMark(contextMark)); + lines.append("\n"); + } + if (problem != null) { + lines.append(problem); + lines.append("\n"); + } + if (problemMark != null) { + lines.append(hiddenSensitiveDataInMark(problemMark)); + lines.append("\n"); + } + + YAMLException yamlException = new YAMLException(lines.toString(), exception.getCause()); Review Comment: Indeed, that is a possibility. I have made updates to address this scenario. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@flink.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
