/***************************************************************
 *
 * Copyright (C) 1990-2018, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 * 
 *	  http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

#include "condor_common.h"
#include "condor_arglist.h"
#include "condor_attributes.h"
#include "condor_getcwd.h"
#include "condor_daemon_core.h"
#include "condor_version.h"
#include "dagman_utils.h"
#include "my_popen.h"
#include "read_multiple_logs.h"
#include "../condor_procapi/processid.h"
#include "../condor_procapi/procapi.h"
#include "tmp_dir.h"
#include "tokener.h"
#include "which.h"

namespace shallow = DagmanShallowOptions;
namespace deep = DagmanDeepOptions;

static void
AppendError(std::string &errMsg, const std::string &newError)
{
	if ( ! errMsg.empty()) errMsg += "; ";
	errMsg += newError;
}

static bool
ImportFilter(const std::string &var, const std::string &val) {
	if (var.find(";") != std::string::npos || val.find(";") != std::string::npos) {
		return false;
	}
	return Env::IsSafeEnvV2Value( val.c_str() );
}

bool
DagmanUtils::writeSubmitFile(DagmanOptions &options, str_list &dagFileAttrLines) const
{
	std::string submitFile = options[shallow::str::SubFile];
	FILE *pSubFile = safe_fopen_wrapper_follow(submitFile.c_str(), "w");
	if ( ! pSubFile) {
		fprintf(stderr, "ERROR: unable to create submit file %s\n",
		        submitFile.c_str() );
		return false;
	}

	std::string executable;
	if (options[shallow::b::RunValgrind]) {
		std::string valgrindPath = which(valgrind_exe);
		if (valgrindPath.empty()) {
			fprintf(stderr, "ERROR: can't find %s in PATH, aborting.\n", valgrind_exe);
			fclose(pSubFile);
			return false;
		} else {
			executable = valgrindPath;
		}
	} else {
		executable = options[deep::str::DagmanPath];
	}

	/*	Set up DAGMan proper jobs getenv filter
	*	Update DAGMAN_MANAGER_JOB_APPEND_GETENV Macro documentation if base
	*	getEnv value changes. -Cole Bollig 2023-02-21
	*/
	std::string getEnv = "CONDOR_CONFIG,_CONDOR_*,PATH,PYTHONPATH,PERL*,PEGASUS_*,TZ,HOME,USER,LANG,LC_ALL";
	auto_free_ptr conf_getenvVars = param("DAGMAN_MANAGER_JOB_APPEND_GETENV");
	if (conf_getenvVars && strcasecmp(conf_getenvVars.ptr(),"true") == MATCH) {
		getEnv = "true";
	} else {
		//Scitoken related variables
		getEnv += ",BEARER_TOKEN,BEARER_TOKEN_FILE,XDG_RUNTIME_DIR";
		//Add user defined via flag vars to getenv
		if ( ! options[deep::str::GetFromEnv].empty()) {
			getEnv += ",";
			getEnv += options[deep::str::GetFromEnv];
		}
		//Add config defined vars to add getenv
		if (conf_getenvVars) { getEnv += ","; getEnv += conf_getenvVars.ptr(); }
	}

	fprintf(pSubFile, "# Filename: %s\n", submitFile.c_str());

	fprintf(pSubFile, "# Generated by condor_submit_dag ");
	for (auto & dagFile : options.dagFiles()) {
		fprintf(pSubFile, "%s ", dagFile.c_str());
	}
	fprintf(pSubFile, "\n");

	fprintf(pSubFile, "universe    = scheduler\n");
	fprintf(pSubFile, "executable  = %s\n", executable.c_str());
	fprintf(pSubFile, "getenv      = %s\n", getEnv.c_str());
	fprintf(pSubFile, "output      = %s\n", options[shallow::str::LibOut].c_str());
	fprintf(pSubFile, "error       = %s\n", options[shallow::str::LibErr].c_str());
	fprintf(pSubFile, "log         = %s\n", options[shallow::str::SchedLog].c_str());
	if ( ! options[deep::str::BatchName].empty()) {
		fprintf(pSubFile, "My.%s = \"%s\"\n", ATTR_JOB_BATCH_NAME,
		        options[deep::str::BatchName].c_str());
	}
	if ( ! options[deep::str::BatchId].empty()) {
		fprintf(pSubFile, "My.%s = \"%s\"\n", ATTR_JOB_BATCH_ID,
		        options[deep::str::BatchId].c_str());
	}
#if !defined ( WIN32 )
	fprintf(pSubFile, "remove_kill_sig = SIGUSR1\n" );
#endif
	fprintf(pSubFile, "My.%s = \"%s =?= $(cluster)\"\n",
	        ATTR_OTHER_JOB_REMOVE_REQUIREMENTS, ATTR_DAGMAN_JOB_ID );

		// ensure DAGMan is automatically requeued by the schedd if it
		// exits abnormally or is killed (e.g., during a reboot)
	const char *defaultRemoveExpr = "(ExitSignal =?= 11 || (ExitCode =!= UNDEFINED && ExitCode >=0 && ExitCode <= 2))";
	std::string removeExpr;
	param(removeExpr, "DAGMAN_ON_EXIT_REMOVE", defaultRemoveExpr);
	fprintf(pSubFile, "# Note: default on_exit_remove expression:\n");
	fprintf(pSubFile, "# %s\n", defaultRemoveExpr);
	fprintf(pSubFile, "# attempts to ensure that DAGMan is automatically\n");
	fprintf(pSubFile, "# requeued by the schedd if it exits abnormally or\n");
	fprintf(pSubFile, "# is killed (e.g., during a reboot).\n");
	fprintf(pSubFile, "on_exit_remove = %s\n", removeExpr.c_str() );

	if (!usingPythonBindings) {
		fprintf(pSubFile, "copy_to_spool = %s\n", options[shallow::b::CopyToSpool] ? "True" : "False" );
	}

	//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	// Be sure to change MIN_SUBMIT_FILE_VERSION in dagman_main.cpp
	// if the arguments passed to condor_dagman change in an
	// incompatible way!!
	//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	ArgList args;

	if (options[shallow::b::RunValgrind]) {
		args.AppendArg("--tool=memcheck");
		args.AppendArg("--leak-check=yes");
		args.AppendArg("--show-reachable=yes");
		args.AppendArg(options[deep::str::DagmanPath].c_str());
	}

		// -p 0 causes DAGMan to run w/o a command socket (see gittrac #4987).
	args.AppendArg("-p");
	args.AppendArg("0");
	args.AppendArg("-f");
	args.AppendArg("-l");
	args.AppendArg(".");
	if (options[shallow::i::DebugLevel] != DEBUG_UNSET) {
		args.AppendArg("-Debug");
		args.AppendArg(std::to_string(options[shallow::i::DebugLevel]));
	}
	args.AppendArg("-Lockfile");
	args.AppendArg(options[shallow::str::LockFile]);

	for (auto & dagFile : options.dagFiles()) {
		args.AppendArg("-Dag");
		args.AppendArg(dagFile);
	}

	if (options[shallow::i::MaxIdle] != 0) {
		args.AppendArg("-MaxIdle");
		args.AppendArg(std::to_string(options[shallow::i::MaxIdle]));
	}

	if (options[shallow::i::MaxJobs] != 0) {
		args.AppendArg("-MaxJobs");
		args.AppendArg(std::to_string(options[shallow::i::MaxJobs]));
	}

	if (options[shallow::i::MaxPre] != 0) {
		args.AppendArg("-MaxPre");
		args.AppendArg(std::to_string(options[shallow::i::MaxPre]));
	}

	if (options[shallow::i::MaxPost] != 0) {
		args.AppendArg("-MaxPost");
		args.AppendArg(std::to_string(options[shallow::i::MaxPost]));
	}

	if (options[shallow::b::PostRun].set()) {
		if (options[shallow::b::PostRun]) {
			args.AppendArg("-AlwaysRunPost");
		} else {
			args.AppendArg("-DontAlwaysRunPost");
		}
	}

	if (options[shallow::b::DoRecovery]) {
		args.AppendArg( "-DoRecov" );
	}

	args.AppendArg("-CsdVersion");
	args.AppendArg(CondorVersion());

	if (options[shallow::b::DumpRescueDag]) {
		args.AppendArg("-DumpRescue");
	}

	if (options[shallow::i::Priority] != 0) {
		args.AppendArg("-Priority");
		args.AppendArg(std::to_string(options[shallow::i::Priority]));
	}

	if ( ! options[shallow::str::SaveFile].empty()) {
		args.AppendArg("-load_save");
		args.AppendArg(options[shallow::str::SaveFile]);
	}

	options.addDeepArgs(args);

	std::string arg_str, args_error;
	if ( ! args.GetArgsStringV1WackedOrV2Quoted(arg_str, args_error)) {
		fprintf(stderr, "Failed to insert arguments: %s", args_error.c_str());
		fclose(pSubFile);
		return false;
	}
	fprintf(pSubFile, "arguments = %s\n", arg_str.c_str());

	//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	// Be sure to change MIN_SUBMIT_FILE_VERSION in dagman_main.cpp
	// if the environment passed to condor_dagman changes in an
	// incompatible way!!
	//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	Env env;
	if (options[deep::b::ImportEnv]) {
		env.Import(ImportFilter);
	}

	for (auto const& kv_pairs : options[deep::slist::AddToEnv]) {
		std::string err_msg;
		env.MergeFromV1RawOrV2Quoted(kv_pairs.c_str(), err_msg);
		if ( ! err_msg.empty()) {
			fprintf(stderr,"Error: Failed to add %s to DAGMan manager jobs environment because %s\n",
			        kv_pairs.c_str(), err_msg.c_str());
			fclose(pSubFile);
			return false;
		}
	}

	env.SetEnv("_CONDOR_DAGMAN_LOG", options[shallow::str::DebugLog].c_str());
	env.SetEnv("_CONDOR_MAX_DAGMAN_LOG=0");
	if ( ! options[shallow::str::ScheddDaemonAdFile].empty()) {
		env.SetEnv("_CONDOR_SCHEDD_DAEMON_AD_FILE",
		           options[shallow::str::ScheddDaemonAdFile].c_str());
	}
	if ( ! options[shallow::str::ScheddAddressFile].empty()) {
		env.SetEnv("_CONDOR_SCHEDD_ADDRESS_FILE",
		           options[shallow::str::ScheddAddressFile].c_str());
	}
	if ( ! options[shallow::str::ConfigFile].empty()) {
		if (access(options[shallow::str::ConfigFile].c_str(), F_OK) != 0) {
			fprintf(stderr, "ERROR: unable to read config file %s (error %d, %s)\n",
			        options[shallow::str::ConfigFile].c_str(), errno, strerror(errno));
			fclose(pSubFile);
			return false;
		}
		env.SetEnv("_CONDOR_DAGMAN_CONFIG_FILE", options[shallow::str::ConfigFile].c_str());
	}

	std::string env_str;
	env.getDelimitedStringV2Quoted(env_str);
	fprintf(pSubFile, "environment = %s\n", env_str.c_str());

	if ( ! options[deep::str::Notification].empty()) {
		fprintf(pSubFile, "notification = %s\n", options[deep::str::Notification].c_str());
	}

		// Append user-specified stuff to submit file...

		// ...first, the insert file, if any...
	if ( ! options[shallow::str::AppendFile].empty()) {
		FILE *aFile = safe_fopen_wrapper_follow(options[shallow::str::AppendFile].c_str(), "r");
		if ( ! aFile) {
			fprintf(stderr, "ERROR: unable to read submit append file (%s)\n",
			        options[shallow::str::AppendFile].c_str());
			fclose(pSubFile);
			return false;
		}

		char *line;
		int lineno = 0;
		while ((line = getline_trim(aFile, lineno)) != nullptr) {
			fprintf(pSubFile, "%s\n", line);
		}

		fclose(aFile);
	}

		// ...now append lines specified in the DAG file...
	for (auto & dagFileAttrLine : dagFileAttrLines) {
		fprintf(pSubFile, "My.%s\n", dagFileAttrLine.c_str());
	}

		// ...now things specified directly on the command line.
	for (auto & appendLine : options[shallow::slist::AppendLines]) {
		fprintf(pSubFile, "%s\n", appendLine.c_str());
	}

	fprintf(pSubFile, "queue\n");
	fclose(pSubFile);

	return true;
}

/** Run condor_submit_dag on the given DAG file.
@param opts: the condor_submit_dag options
@param dagFile: the DAG file to process
@param directory: the directory from which the DAG file should
	be processed (ignored if NULL)
@param priority: the priority of this DAG
@param isRetry: whether this is a retry
@return 0 if successful, 1 if failed
*/
int
DagmanUtils::runSubmitDag(const DagmanOptions &options, const char *dagFile,
                          const char *directory, int priority, bool isRetry)
{
	int result = 0;

		// Change to the appropriate directory if necessary.
	TmpDir tmpDir;
	std::string errMsg;
	if (directory) {
		if ( ! tmpDir.Cd2TmpDir(directory, errMsg)) {
			fprintf(stderr, "Error (%s) changing to node directory\n",
			        errMsg.c_str() );
			return 1;
		}
	}

		// Build up the command line for the recursive run of
		// condor_submit_dag.  We need -no_submit so we don't
		// actually run the subdag now; we need -update_submit
		// so the lower-level .condor.sub file will get
		// updated, in case it came from an earlier version
		// of condor_submit_dag.
	ArgList args;
	args.AppendArg("condor_submit_dag");
	args.AppendArg("-no_submit");
	args.AppendArg("-update_submit");

	if (options[deep::b::Force] && !isRetry) {
		args.AppendArg("-force");
	}

	if (priority != 0) {
		args.AppendArg("-Priority");
		args.AppendArg(std::to_string(priority));
	}

	options.addDeepArgs(args, false);

	args.AppendArg(dagFile);

	std::string cmdLine;
	args.GetArgsStringForDisplay(cmdLine);
	dprintf(D_ALWAYS, "Recursive submit command: <%s>\n",
	        cmdLine.c_str() );

		// Now actually run the command.
	int retval = my_system(args);
	if (retval != 0) {
		dprintf(D_ALWAYS, "ERROR: condor_submit_dag -no_submit failed on DAG file %s.\n",
		        dagFile);
		result = 1;
	}

		// Change back to the directory we started from.
	if ( ! tmpDir.Cd2MainDir(errMsg)) {
		dprintf(D_ALWAYS, "Error (%s) changing back to original directory\n",
		        errMsg.c_str() );
	}

	return result;
}

//---------------------------------------------------------------------------
/** Set up things in deep and shallow options that aren't directly specified
	on the command line.
	@param optionss: the DAGMan options
	@return true on success and false on failure
*/
bool
DagmanUtils::setUpOptions(DagmanOptions &options, str_list &dagFileAttrLines)
{
	std::string primaryDag = options.primaryDag();
	options[shallow::str::LibOut] = primaryDag + ".lib.out";
	options[shallow::str::LibErr] = primaryDag + ".lib.err";

	if ( ! options[deep::str::OutfileDir].empty()) {
		options[shallow::str::DebugLog] = options[deep::str::OutfileDir] + DIR_DELIM_STRING +
		                                  condor_basename(primaryDag.c_str());
	} else {
		options[shallow::str::DebugLog] = primaryDag;
	}
	options[shallow::str::DebugLog] += ".dagman.out";
	options[shallow::str::SchedLog] = primaryDag + ".dagman.log";
	options[shallow::str::SubFile] = primaryDag + DAG_SUBMIT_FILE_SUFFIX;

	std::string rescueDagBase;

		// If we're running each DAG in its own directory, write any rescue
		// DAG to the current directory, to avoid confusion (since the
		// rescue DAG must be run from the current directory).
	if (options[deep::b::UseDagDir]) {
		if ( ! condor_getcwd(rescueDagBase)) {
			fprintf(stderr, "ERROR: unable to get cwd: %d, %s\n",
			        errno, strerror(errno));
			return false;
		}
		rescueDagBase += DIR_DELIM_STRING;
		rescueDagBase += condor_basename(primaryDag.c_str());
	} else {
		rescueDagBase = primaryDag;
	}

		// If we're running multiple DAGs, put "_multi" in the rescue
		// DAG name to indicate that the rescue DAG is for *all* of
		// the DAGs we're running.
	if (options.isMultiDag()) {
		rescueDagBase += "_multi";
	}
	options[shallow::str::RescueFile] = rescueDagBase + ".rescue";

	options[shallow::str::LockFile] = primaryDag + ".lock";

	if (options[deep::str::DagmanPath].empty()) {
		options[deep::str::DagmanPath] = which(dagman_exe);
	}

	if (options[deep::str::DagmanPath].empty()) {
		fprintf(stderr, "ERROR: can't find %s in PATH, aborting.\n",
		        dagman_exe );
		return false;
	}
	std::string msg;
	if ( ! processDagCommands(options, dagFileAttrLines, msg)) {
		fprintf(stderr, "ERROR: %s\n", msg.c_str());
		return false;
	}

	return true;
}

/** Read the DAG files for DAG commands that need to be parsed
	before the submission of the DAGMan job proper because the
	commands effect the produced .condor.sub file
	@param options: DAGMan options class
	@param attrLines: list of strings of attributes to be added
	                  to the DAGMan job propers classad
	@param errMsg: error message
	@return true if the operation succeeded; otherwise false
*/
bool
DagmanUtils::processDagCommands(DagmanOptions &options, str_list &attrLines, std::string &errMsg)
{
	bool result = true;
	// Note: destructor will change back to original directory.
	TmpDir dagDir;
	std::set<std::string> configFiles;

	for (auto & dagFile : options.dagFiles()) {
		std::string newDagFile;
		// Switch to DAG Dir if needed
		if (options[deep::b::UseDagDir]) {
			std::string tmpErrMsg;
			if ( ! dagDir.Cd2TmpDirFile(dagFile.c_str(), tmpErrMsg)) {
				errMsg = "Unable to change to DAG directory " + tmpErrMsg;
				return false;
			}
			newDagFile = condor_basename(dagFile.c_str());
		} else {
			newDagFile = dagFile.c_str();
		}

		// Note: destructor will close file.
		MultiLogFiles::FileReader reader;
		errMsg = reader.Open( newDagFile );
		if ( ! errMsg.empty()) {
			return false;
		}

		//Read DAG file
		std::string logicalLine;
		while (reader.NextLogicalLine(logicalLine)) {
			if ( ! logicalLine.empty()) {
				trim(logicalLine);
				StringTokenIterator tokens(logicalLine);
				const char* cmd = tokens.first();

				// Parse CONFIG command
				if (strcasecmp(cmd, "CONFIG") == MATCH) {
					const char* newFile = tokens.remain();
					while (newFile && isspace(*newFile) && *newFile != '\0') { newFile++; }
					if ( ! newFile || *newFile == '\0') {
						AppendError(errMsg, "Improperly-formatted file: value missing after keyword CONFIG");
						result = false;
					} else {
						std::string conf(newFile), tmpErr;
						if (MakePathAbsolute(conf, tmpErr)) {
							configFiles.insert(conf);
						} else {
							AppendError(errMsg, tmpErr);
							result = false;
						}
					}

				// Parse SET_JOB_ATTR command
				} else if (strcasecmp(cmd, "SET_JOB_ATTR") == MATCH) {
					const char* attr = tokens.remain();
					while (attr && isspace(*attr) && *attr != '\0') { attr++; }
					if (!attr || *attr == '\0') {
						AppendError(errMsg, "Improperly-formatted file: value missing after keyword SET_JOB_ATTR");
						result = false;
					} else {
						attrLines.emplace_back(attr);
					}

				// Parse ENV command
				} else if (strcasecmp(cmd, "ENV") == MATCH) {
					const char* type = tokens.next();
					// Parse GET option
					if (strcasecmp(type, "GET") == MATCH) {
						const char* remain = tokens.remain();
						while (remain && isspace(*remain) && *remain != '\0') { remain++; }
						if (!remain || *remain == '\0') {
							AppendError(errMsg, "Improperly-formatted file: environment variables missing after ENV GET");
							result = false;
						} else {
							StringTokenIterator vars(remain);
							for (auto& var : vars) {
								options.append("GetFromEnv", var);
							}
						}
					// Parse SET option
					} else if (strcasecmp(type, "SET") == MATCH) {
						const char* info = tokens.remain();
						while (info && isspace(*info) && *info != '\0') { info++; }
						if (!info || *info == '\0') {
							AppendError(errMsg, "Improperly-formatted file: environment variables missing after ENV SET");
							result = false;
						} else {
							std::string kv_pairs(info);
							trim(kv_pairs);
							options.extend("AddToEnv", kv_pairs);
						}
					// Else error
					} else {
						AppendError(errMsg, "Improperly-formatted file: sub-command (SET or GET) missing after keyword ENV");
						result = false;
					}
				}
			}
		}

		reader.Close();

		// Switch back to original directory
		std::string tmpErrMsg;
		if ( ! dagDir.Cd2MainDir(tmpErrMsg)) {
			AppendError(errMsg, "Unable to change to original directory " + tmpErrMsg);
			result = false;
		}
	}

	// Verify config files (only 1 file given)
	if ( ! configFiles.empty()) {
		if (configFiles.size() == 1) {
			std::string conf = *configFiles.begin();
			// Check new config file for match with cli passed file
			if (options[shallow::str::ConfigFile].empty()) {
				options[shallow::str::ConfigFile] = conf;
			} else if (options[shallow::str::ConfigFile] != conf) {
				AppendError(errMsg, "Conflicting DAGMan config files specified: " +
				                    options[shallow::str::ConfigFile] + " and " + conf);
				result = false;
			}
		} else {
			size_t num_conf = configFiles.size();
			AppendError(errMsg, "Multiple (" + std::to_string(num_conf) +
			                    ") configuration files provided.");
			result = false;
		}
	}

	return result;
}

/** Make the given path into an absolute path, if it is not already.
	@param filePath: the path to make absolute (filePath is changed)
	@param errMsg: a std::string to receive any error message.
	@return true if the operation succeeded; otherwise false
*/
bool
DagmanUtils::MakePathAbsolute(std::string &filePath, std::string &errMsg)
{
	bool result = true;

	if ( ! fullpath(filePath.c_str())) {
		std::string currentDir;
		if ( ! condor_getcwd(currentDir)) {
			formatstr(errMsg, "condor_getcwd() failed with errno %d (%s) at %s:%d",
			          errno, strerror(errno), __FILE__, __LINE__);
			result = false;
		}
		filePath = currentDir + DIR_DELIM_STRING + filePath;
	}

	return result;
}

/** Finds the number of the last existing rescue DAG file for the
	given "primary" DAG.
	@param primaryDagFile The primary DAG file name
	@param multiDags Whether we have multiple DAGs
	@param maxRescueDagNum the maximum legal rescue DAG number
	@return The number of the last existing rescue DAG (0 if there
		is none)
*/
int
DagmanUtils::FindLastRescueDagNum(const std::string &primaryDagFile, bool multiDags, int maxRescueDagNum)
{
	int lastRescue = 0;

	for ( int test = 1; test <= maxRescueDagNum; test++ ) {
		std::string testName = RescueDagName(primaryDagFile, multiDags, test);
		if (access(testName.c_str(), F_OK) == 0) {
			if (test > lastRescue + 1) {
					// This should probably be a fatal error if
					// DAGMAN_USE_STRICT is set, but I'm avoiding
					// that for now because the fact that this code
					// is used in both condor_dagman and condor_submit_dag
					// makes that harder to implement. wenger 2011-01-28
				dprintf(D_ALWAYS, "Warning: found rescue DAG number %d, but not rescue DAG number %d\n",
				        test, test - 1);
			}
			lastRescue = test;
		}
	}
	
	if (lastRescue >= maxRescueDagNum) {
		dprintf(D_ALWAYS, "Warning: FindLastRescueDagNum() hit maximum rescue DAG number: %d\n",
		        maxRescueDagNum);
	}

	return lastRescue;
}

/** Creates a rescue DAG name, given a primary DAG name and rescue
	DAG number
	@param primaryDagFile The primary DAG file name
	@param multiDags Whether we have multiple DAGs
	@param rescueDagNum The rescue DAG number
	@return The full name of the rescue DAG
*/
std::string
DagmanUtils::RescueDagName(const std::string& primaryDagFile, bool multiDags, int rescueDagNum)
{
	ASSERT( rescueDagNum >= 1 );

	std::string fileName(primaryDagFile);
	if ( multiDags ) {
		fileName += "_multi";
	}
	fileName += ".rescue";
	formatstr_cat(fileName, "%.3d", rescueDagNum);

	return fileName;
}

/** Renames all rescue DAG files for this primary DAG after the
	given one (as long as the numbers are contiguous).	For example,
	if rescueDagNum is 3, we will rename .rescue4, .rescue5, etc.
	@param primaryDagFile The primary DAG file name
	@param multiDags Whether we have multiple DAGs
	@param rescueDagNum The rescue DAG number to rename *after*
	@param maxRescueDagNum the maximum legal rescue DAG number
*/
void
DagmanUtils::RenameRescueDagsAfter(const std::string& primaryDagFile, bool multiDags, int rescueDagNum, int maxRescueDagNum)
{
		// Need to allow 0 here so condor_submit_dag -f can rename all
		// rescue DAGs.
	ASSERT( rescueDagNum >= 0 );

	dprintf(D_ALWAYS, "Renaming rescue DAGs newer than number %d\n", rescueDagNum);

	int firstToDelete = rescueDagNum + 1;
	int lastToDelete = FindLastRescueDagNum(primaryDagFile, multiDags, maxRescueDagNum);

	for (int rescueNum = firstToDelete; rescueNum <= lastToDelete; rescueNum++) {
		std::string rescueDagName = RescueDagName(primaryDagFile, multiDags, rescueNum);
		dprintf(D_ALWAYS, "Renaming %s\n", rescueDagName.c_str());
		std::string newName = rescueDagName + ".old";
		// Unlink here to be safe on Windows.
		tolerant_unlink(newName);
		if (rename(rescueDagName.c_str(), newName.c_str()) != 0) {
			EXCEPT("Fatal error: unable to rename old rescue file %s: error %d (%s)\n",
			       rescueDagName.c_str(), errno, strerror(errno));
		}
	}
}

/** Attempts to unlink the given file, and prints an appropriate error
	message if this fails (but doesn't return an error, so only call
	this if a failure of the unlink is okay).
	@param pathname The path of the file to unlink
*/
void
DagmanUtils::tolerant_unlink(const std::string &pathname)
{
	if (unlink(pathname.c_str()) != 0) {
		if (errno == ENOENT) {
			dprintf(D_SYSCALLS, "Warning: failure (%d (%s)) attempting to unlink file %s\n",
			        errno, strerror(errno), pathname.c_str());
		} else {
			dprintf(D_ALWAYS, "Error (%d (%s)) attempting to unlink file %s\n",
			        errno, strerror(errno), pathname.c_str());
		}
	}
}

//---------------------------------------------------------------------------
bool 
DagmanUtils::fileExists(const std::string &strFile)
{
	int fd = safe_open_wrapper_follow(strFile.c_str(), O_RDONLY);
	if (fd == -1)
		return false;
	close(fd);
	return true;
}

//---------------------------------------------------------------------------
bool 
DagmanUtils::ensureOutputFilesExist(const DagmanOptions &options)
{
	int maxRescueDagNum = param_integer("DAGMAN_MAX_RESCUE_NUM", MAX_RESCUE_DAG_DEFAULT, 0, ABS_MAX_RESCUE_DAG_NUM);

	if (options[deep::i::DoRescueFrom] > 0) {
		std::string rescueDagName = RescueDagName(options.primaryDag(), options.isMultiDag(), options[deep::i::DoRescueFrom]);
		if ( ! fileExists(rescueDagName)) {
			fprintf(stderr, "-dorescuefrom %d specified, but rescue DAG file %s does not exist!\n",
			                options[deep::i::DoRescueFrom], rescueDagName.c_str());
			return false;
		}
	}

		// Get rid of the halt file (if one exists).
	tolerant_unlink(HaltFileName(options.primaryDag()));

	if (options[deep::b::Force]) {
		tolerant_unlink(options[shallow::str::SubFile]);
		tolerant_unlink(options[shallow::str::SchedLog]);
		tolerant_unlink(options[shallow::str::LibOut]);
		tolerant_unlink(options[shallow::str::LibErr]);
		RenameRescueDagsAfter(options.primaryDag(), options.isMultiDag(), 0, maxRescueDagNum);
	}

		// Check whether we're automatically running a rescue DAG -- if
		// so, allow things to continue even if the files generated
		// by condor_submit_dag already exist.
	bool autoRunningRescue = false;
	if (options[deep::b::AutoRescue]) {
		int rescueDagNum = FindLastRescueDagNum(options.primaryDag(), options.isMultiDag(), maxRescueDagNum);
		if (rescueDagNum > 0) {
			printf("Running rescue DAG %d\n", rescueDagNum);
			autoRunningRescue = true;
		}
	}

	bool bHadError = false;
		// If not running a rescue DAG, check for existing files
		// generated by condor_submit_dag...
	if ( ! autoRunningRescue && options[deep::i::DoRescueFrom] < 1 &&
		 ! options[deep::b::UpdateSubmit] && options[shallow::str::SaveFile].empty()) {
			if (fileExists(options[shallow::str::SubFile])) {
				fprintf(stderr, "ERROR: \"%s\" already exists.\n",
				        options[shallow::str::SubFile].c_str());
				bHadError = true;
			}
			if (fileExists(options[shallow::str::LibOut])) {
				fprintf(stderr, "ERROR: \"%s\" already exists.\n",
				        options[shallow::str::LibOut].c_str());
				bHadError = true;
			}
			if (fileExists(options[shallow::str::LibErr])) {
				fprintf(stderr, "ERROR: \"%s\" already exists.\n",
				        options[shallow::str::LibErr].c_str());
				bHadError = true;
			}
			if (fileExists(options[shallow::str::SchedLog])) {
				fprintf(stderr, "ERROR: \"%s\" already exists.\n",
				        options[shallow::str::SchedLog].c_str());
				bHadError = true;
			}
	}

		// This is checking for the existance of an "old-style" rescue
		// DAG file.
	if ( ! options[deep::b::AutoRescue] && options[deep::i::DoRescueFrom] < 1 &&
		 fileExists(options[shallow::str::RescueFile])) {
			fprintf(stderr, "ERROR: \"%s\" already exists.\n",
			        options[shallow::str::RescueFile].c_str());
			fprintf(stderr, "\tYou may want to resubmit your DAG using that file, instead of \"%s\"\n",
			        options.primaryDag().c_str());
			fprintf(stderr, "\tLook at the HTCondor manual for details about DAG rescue files.\n");
			fprintf(stderr, "\tPlease investigate and either remove \"%s\",\n",
			        options[shallow::str::RescueFile].c_str());
			fprintf(stderr, "\tor use it as the input to condor_submit_dag.\n");
			bHadError = true;
	}

	if (bHadError) {
		fprintf(stderr, "\nSome file(s) needed by %s already exist. Either:\n- Rename them\n",
		        dagman_exe );
		if( ! usingPythonBindings) {
			fprintf(stderr, "- Use the \"-f\" option to force them to be overwritten\n");
			fprintf(stderr, "\tor\n- Use the \"-update_submit\" option to update the submit file and continue.\n" );
		}
		else {
			fprintf(stderr, "\tor\n- Set the { \"force\" : True } option to force them to be overwritten.\n");
		}
		return false;
	}

	return true;
}

//-----------------------------------------------------------------------------
int 
DagmanUtils::popen (ArgList &args) {
	std::string cmd; // for debug output
	args.GetArgsStringForDisplay(cmd);
	dprintf(D_ALWAYS, "Running: %s\n", cmd.c_str());

	FILE *fp = my_popen( args, "r", MY_POPEN_OPT_WANT_STDERR );

	int r = 0;
	if (fp == nullptr || (r = my_pclose(fp) & 0xff) != 0) {
		dprintf(D_ERROR, "Warning: failure: %s\n", cmd.c_str());
		if(fp != nullptr) {
			dprintf(D_ALWAYS, "\t(my_pclose() returned %d (errno %d, %s))\n",
			        r, errno, strerror(errno));
		} else {
			dprintf(D_ALWAYS, "\t(my_popen() returned NULL (errno %d, %s))\n",
			        errno, strerror(errno));
			r = -1;
		}
	}
	return r;
}

//-----------------------------------------------------------------------------
int
DagmanUtils::create_lock_file(const char *lockFileName, bool abortDuplicates) {
	int result = 0;

	FILE *fp = safe_fopen_wrapper_follow(lockFileName, "w");
	if (fp == nullptr) {
		dprintf(D_ALWAYS, "ERROR: could not open lock file %s for writing.\n", lockFileName);
		result = -1;
	}

	// Create the ProcessId object.
	ProcessId *procId = nullptr;
	if (result == 0 && abortDuplicates) {
		int status;
		int precision_range = 1;
		if (ProcAPI::createProcessId(daemonCore->getpid(), procId, status, &precision_range) != PROCAPI_SUCCESS) {
			dprintf(D_ALWAYS, "ERROR: ProcAPI::createProcessId() failed; %d\n", status);
			result = -1;
		}
	}

	// Write out the ProcessId object.
	if (result == 0 && abortDuplicates) {
		if (procId->write(fp) != ProcessId::SUCCESS) {
			dprintf(D_ALWAYS, "ERROR: ProcessId::write() failed\n");
			result = -1;
		}
	}

	// Confirm the ProcessId object's uniqueness.
	if (result == 0 && abortDuplicates) {
		int status;
		if (ProcAPI::confirmProcessId(*procId, status) != PROCAPI_SUCCESS) {
			dprintf(D_ERROR, "Warning: ProcAPI::confirmProcessId() failed; %d\n",
			        status);
		} else if ( ! procId->isConfirmed()) {
			dprintf(D_ERROR, "Warning: ProcessId not confirmed unique\n");
		} else if (procId->writeConfirmationOnly(fp) != ProcessId::SUCCESS) {
			dprintf(D_ERROR, "ERROR: ProcessId::writeConfirmationOnly() failed\n");
			        result = -1;
		}
	}

	delete procId;

	if (fp != nullptr) {
		if (fclose(fp) != 0) {
			dprintf(D_ALWAYS, "ERROR: closing lock file failed with errno %d (%s)\n",
			        errno, strerror(errno));
		}
	}

	return result;
}

//-----------------------------------------------------------------------------
int
DagmanUtils::check_lock_file(const char *lockFileName) {
	int result = 0;

	FILE *fp = safe_fopen_wrapper_follow(lockFileName, "r");
	if (fp == nullptr) {
		dprintf(D_ALWAYS, "ERROR: could not open lock file %s for reading.\n",
		        lockFileName);
		result = -1;
	}

	ProcessId *procId = nullptr;
	if (result != -1) {
		int status;
		procId = new ProcessId(fp, status);
		if (status != ProcessId::SUCCESS) {
			dprintf(D_ALWAYS, "ERROR: unable to create ProcessId object from lock file %s\n",
			        lockFileName);
			result = -1;
		}
	}

	if (result != -1) {
		int status;
		if (ProcAPI::isAlive(*procId, status) != PROCAPI_SUCCESS) {
			dprintf(D_ALWAYS, "ERROR: failed to determine whether DAGMan that wrote lock file is alive\n");
			result = -1;
		} else if (status == PROCAPI_ALIVE) {
			dprintf(D_ALWAYS, "Duplicate DAGMan PID %d is alive; this DAGMan should abort.\n",
			        procId->getPid());
			result = 1;

		} else if (status == PROCAPI_DEAD) {
			dprintf(D_ALWAYS, "Duplicate DAGMan PID %d is no longer alive; this DAGMan should continue.\n",
			        procId->getPid());
			result = 0;

		} else if (status == PROCAPI_UNCERTAIN) {
			dprintf(D_ALWAYS, "Duplicate DAGMan PID %d *may* be alive; this DAGMan is continuing, "
			                  "but this will cause problems if the duplicate DAGMan is alive.\n",
			                  procId->getPid());
			result = 0;

		} else {
			EXCEPT("Illegal ProcAPI::isAlive() status value: %d", status);
		}
	}

	delete procId;

	if (fp != nullptr) {
		if (fclose(fp) != 0) {
			dprintf(D_ALWAYS, "ERROR: closing lock file failed with errno %d (%s)\n",
			        errno, strerror(errno));
		}
	}

	return result;
}

// Add a DAG file to DAGMan options
void DagmanOptions::addDAGFile(std::string& dagFile) {
	using namespace shallow;

	if (primaryDag().empty()) {
		shallow.stringOpts[str::PrimaryDagFile] = dagFile;
	}

	str_list &dag_files = shallow.slistOpts[slist::DagFiles];
	dag_files.push_back(dagFile);
	if ( ! is_MultiDag) { is_MultiDag = dag_files.size() > 1; }
}

// Add DAGMan options deep args to an ArgList
void DagmanOptions::addDeepArgs(ArgList& args, bool inWriteSubmit) const {
	using namespace deep;
	const auto &self = *this;
	if (self[b::Verbose]) {
		args.AppendArg("-verbose");
	}

	if ( ! self[str::Notification].empty()) {
		args.AppendArg( "-notification" );
		if(self[b::SuppressNotification]) {
			args.AppendArg("never");
		} else {
			args.AppendArg(self[str::Notification]);
		}
	}

	if ( ! self[str::DagmanPath].empty()) {
		args.AppendArg("-dagman");
		args.AppendArg(self[str::DagmanPath]);
	}

	if (self[b::UseDagDir]) {
		args.AppendArg("-UseDagDir");
	}

	if ( ! self[str::OutfileDir].empty()) {
		args.AppendArg("-outfile_dir");
		args.AppendArg(self[str::OutfileDir]);
	}

	args.AppendArg("-AutoRescue");
	args.AppendArg(std::to_string(self[b::AutoRescue]));

	if (inWriteSubmit || self[i::DoRescueFrom]) {
		args.AppendArg("-DoRescueFrom");
		args.AppendArg(std::to_string(self[i::DoRescueFrom]));
	}

	if (self[b::AllowVersionMismatch]) {
		args.AppendArg("-AllowVersionMismatch");
	}

	if (self[b::ImportEnv]) {
		args.AppendArg("-import_env");
	}

	if ( ! self[str::GetFromEnv].empty()) {
		args.AppendArg("-include_env");
		args.AppendArg(self[str::GetFromEnv]);
	}

	for (auto &kv_pair : self[slist::AddToEnv]) {
		args.AppendArg("-insert_env");
		args.AppendArg(kv_pair);
	}

	if (self[b::Recurse]) {
		args.AppendArg("-do_recurse");
	}

	if(self[b::SuppressNotification]) {
		args.AppendArg("-suppress_notification");
	} else if(self[b::SuppressNotification].set()) {
		args.AppendArg("-dont_suppress_notification");
	}

	if (inWriteSubmit) {
		if (self[b::Force]) {
			args.AppendArg("-force");
		}

		if (self[b::UpdateSubmit]) {
			args.AppendArg("-update_submit");
		}
	}

}

// Automagically set DAGMan option that is a boolean
SetDagOpt DagmanOptions::set(const char* opt, bool value) {
	if ( ! opt || *opt == '\0') { return SetDagOpt::NO_KEY; }
	auto s_bool_key = shallow::b::_from_string_nocase_nothrow(opt);
	if (s_bool_key) {
		shallow.boolOpts[*s_bool_key] = value;
		return SetDagOpt::SUCCESS;
	}
	auto d_bool_key = deep::b::_from_string_nocase_nothrow(opt);
	if (d_bool_key) {
		deep.boolOpts[*d_bool_key] = value;
		return SetDagOpt::SUCCESS;
	}
	return SetDagOpt::KEY_DNE;
}

// Automagically set an DAGMan option that is an integer
SetDagOpt DagmanOptions::set(const char* opt, int value){
	if ( ! opt || *opt == '\0') { return SetDagOpt::NO_KEY; }
	auto s_int_key = shallow::i::_from_string_nocase_nothrow(opt);
	if (s_int_key) {
		shallow.intOpts[*s_int_key] = value;
		return SetDagOpt::SUCCESS;
	}
	auto d_int_key = deep::i::_from_string_nocase_nothrow(opt);
	if (d_int_key) {
		deep.intOpts[*d_int_key] = value;
		return SetDagOpt::SUCCESS;
	}
	return SetDagOpt::KEY_DNE;
}

// Convert string to bool. If number then assume greater than 0 is true
static bool str2bool(const std::string& value) {
	std::string check = value;
	lower_case(check);
	if (check == "false") { return false; }
	if (check == "true") { return true; }
	return std::stoi(check) > 0;
}

SetDagOpt DagmanOptions::set(const char* opt, const char* value) {
	if ( ! value || *value == '\0') { return SetDagOpt::NO_VALUE; }
	std::string v(value);
	return set(opt, v);
}

// Automagically set a DAGMan option from a string value
SetDagOpt DagmanOptions::set(const char* opt, const std::string& value) {
	if ( ! opt || *opt == '\0') { return SetDagOpt::NO_KEY; }
	if (value.empty()) { return SetDagOpt::NO_VALUE; }

	// Check option in shallow option enums
	auto s_str_key = shallow::str::_from_string_nocase_nothrow(opt);
	if (s_str_key) {
		shallow.stringOpts[*s_str_key] = value;
		return SetDagOpt::SUCCESS;
	}
	auto s_slist_key = shallow::slist::_from_string_nocase_nothrow(opt);
	if (s_slist_key) {
		shallow.slistOpts[*s_slist_key].push_back(value);
		return SetDagOpt::SUCCESS;
	}
	auto s_bool_key = shallow::b::_from_string_nocase_nothrow(opt);
	if (s_bool_key) {
		try {
			shallow.boolOpts[*s_bool_key] = str2bool(value);
			return SetDagOpt::SUCCESS;
		} catch (...) { return SetDagOpt::INVALID_VALUE; }
	}
	auto s_int_key = shallow::i::_from_string_nocase_nothrow(opt);
	if (s_int_key) {
		try {
			shallow.intOpts[*s_int_key] = std::stoi(value);
			return SetDagOpt::SUCCESS;
		} catch(...) { return SetDagOpt::INVALID_VALUE; }
	}

	// Check option in deep option enums
	auto d_str_key = deep::str::_from_string_nocase_nothrow(opt);
	if (d_str_key) {
		deep.stringOpts[*d_str_key] = value;
		return SetDagOpt::SUCCESS;
	}
	auto d_slist_key = deep::slist::_from_string_nocase_nothrow(opt);
	if (d_slist_key) {
		deep.slistOpts[*d_slist_key].push_back(value);
		return SetDagOpt::SUCCESS;
	}
	auto d_bool_key = deep::b::_from_string_nocase_nothrow(opt);
	if (d_bool_key) {
		try {
			deep.boolOpts[*d_bool_key] = str2bool(value);
			return SetDagOpt::SUCCESS;
		} catch(...) { return SetDagOpt::INVALID_VALUE; }
	}
	auto d_int_key = deep::i::_from_string_nocase_nothrow(opt);
	if (d_int_key) {
		try {
			deep.intOpts[*d_int_key] = std::stoi(value);
			return SetDagOpt::SUCCESS;
		} catch(...) { return SetDagOpt::INVALID_VALUE; }
	}
	return SetDagOpt::KEY_DNE;
}


/*
*	The DagmanOption::append() function takes a string DAGMan
*	option and appends the provided value to it to create or add
*	to a delimited list. Base delimiter is a comma
*/
SetDagOpt DagmanOptions::append(const char* opt, const char* value, const char delim) {
	if ( ! value || *value == '\0') { return SetDagOpt::NO_VALUE; }
	std::string v(value);
	return append(opt, v, delim);
}

SetDagOpt DagmanOptions::append(const char* opt, const std::string& value, const char delim) {
	if ( ! opt || *opt == '\0') { return SetDagOpt::NO_KEY; }
	if (value.empty()) { return SetDagOpt::NO_VALUE; }

	auto s_str_key = shallow::str::_from_string_nocase_nothrow(opt);
	if (s_str_key) {
		if ( ! shallow.stringOpts[*s_str_key].empty()) {
			shallow.stringOpts[*s_str_key] += delim;
		}
		shallow.stringOpts[*s_str_key] += value;
		return SetDagOpt::SUCCESS;
	}

	auto d_str_key = deep::str::_from_string_nocase_nothrow(opt);
	if (d_str_key) {
		if ( ! deep.stringOpts[*d_str_key].empty()) {
			deep.stringOpts[*d_str_key] += delim;
		}
		deep.stringOpts[*d_str_key] += value;
		return SetDagOpt::SUCCESS;
	}

	return SetDagOpt::KEY_DNE;
}

/*
*	The DagmanOptions::extend() takes a string value to append to a
*	list of strings associated with the specified DAGMan option.
*	Note: DagmanOptions::set() for an slist option does the same
*	behavior, but this is here to be explicit and reduce confusion.
*	- Cole Bollig: 2023-12-29
*/
SetDagOpt DagmanOptions::extend(const char* opt, const char* value) {
	if ( ! value || *value == '\0') { return SetDagOpt::NO_VALUE; }
	std::string v(value);
	return extend(opt, v);
}

SetDagOpt DagmanOptions::extend(const char* opt, const std::string& value) {
	if ( ! opt || *opt == '\0') { return SetDagOpt::NO_KEY; }
	if (value.empty()) { return SetDagOpt::NO_VALUE; }

	auto s_slist_key = shallow::slist::_from_string_nocase_nothrow(opt);
	if (s_slist_key) {
		shallow.slistOpts[*s_slist_key].push_back(value);
		return SetDagOpt::SUCCESS;
	}
	auto d_slist_key = deep::slist::_from_string_nocase_nothrow(opt);
	if (d_slist_key) {
		deep.slistOpts[*d_slist_key].push_back(value);
		return SetDagOpt::SUCCESS;
	}

	return SetDagOpt::KEY_DNE;
}

/*
*	Option Value type represents the type of input to set
*	into an option. While these types can be passed in string
*	representation, we convert the string to the corrent type
*	based on option. Current Option type -> Value type are:
*	    1. STR -> string
*	    2. SLIST -> string
*	    3. B -> boolean
*	    4. I -> integer
*	- Cole Bollig 2023-12-20
*/
std::string DagmanOptions::OptValueType(std::string& opt) {
	if (shallow::b::_from_string_nocase_nothrow(opt.c_str()) ||
		deep::b::_from_string_nocase_nothrow(opt.c_str())) {
			return "bool";
	}

	if (shallow::i::_from_string_nocase_nothrow(opt.c_str()) ||
		deep::i::_from_string_nocase_nothrow(opt.c_str())) {
			return "integer";
	}

	return "string";
}

