/***************************************************************
 *
 * Copyright (C) 1990-2014, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

#include "condor_common.h"
#include "condor_classad.h"
#include "condor_config.h"
#include "directory.h"
#include "docker_proc.h"
#include "starter.h"
#include "condor_holdcodes.h"
#include "docker-api.h"
#include "condor_daemon_client.h"
#include "condor_daemon_core.h"
#include "classad_helpers.h"
#include "ToE.h"

#ifdef HAVE_SCM_RIGHTS_PASSFD
#include "shared_port_scm_rights.h"
#include "fdpass.h"
#endif

extern Starter *Starter;

static void buildExtraVolumes(std::list<std::string> &extras, ClassAd &machAd, ClassAd &jobAd);

static bool handleFTL(int error) {
	if (error != 0) {
		// dprintf( D_ALWAYS, "Failed to launch Docker universe job (%s).\n", error );
	}

	//
	// If we failed to launch the job (as opposed to aborted the takeoff
	// because there was something wrong with the payload), we also need
	// to force the startd to advertise this fact so other jobs can avoid
	// this machine.
	//
	const char * nullString = nullptr;
	DCStartd startd( nullString);
	if( ! startd.locate() ) {
		dprintf( D_ERROR, "Unable to locate startd: %s\n", startd.error() );
		return false;
	}

	//
	// The startd will update the list 'OfflineUniverses' for us.
	//
	ClassAd update;
	if (error) {
		update.Assign( ATTR_HAS_DOCKER, false );
		update.Assign( "DockerOfflineReason", error );
	} else {
		update.Assign( ATTR_HAS_DOCKER, true );
	}

	ClassAd reply;
	if( ! startd.updateMachineAd( &update, &reply ) ) {
		dprintf( D_ERROR, "Unable to update machine ad: %s\n", startd.error() );
		return false;
	}

	return true;
}


//
// TODO: Allow the use of HTCondor file-transfer to provide the image.
// May require understanding "self-hosting".
//

//
// TODO: Leverage the procd to gather usage information; Docker uses
// the full container ID as (part of) the cgroup identifier(s).
//

DockerProc::DockerProc( ClassAd * jobAd ) : VanillaProc( jobAd ), updateTid(-1), memUsage(0), max_memUsage(0), netIn(0), netOut(0), userCpu(0), sysCpu(0), waitForCreate(false), execReaperId(-1), shouldAskForServicePorts(false) { }

DockerProc::~DockerProc() { 
	if ( daemonCore && daemonCore->SocketIsRegistered(&listener)) {
		daemonCore->Cancel_Socket(&listener);
	}
}

int DockerProc::StartJob() {
	std::string imageID;

	// Not really ready for ssh-to-job'ing until we start the container
	Starter->SetJobEnvironmentReady(false);

	if( ! JobAd->LookupString( ATTR_DOCKER_IMAGE, imageID ) ) {
		if (! JobAd->LookupString(ATTR_CONTAINER_IMAGE, imageID)) {
			dprintf( D_ERROR, "Neither %s nor %s defined in job ad, unable to start job.\n", ATTR_DOCKER_IMAGE, ATTR_CONTAINER_IMAGE);
			return FALSE;
		}
	}

	if (starts_with(imageID, "docker://")) {
		imageID = imageID.substr(9);
	}

	std::string command;
	JobAd->LookupString(ATTR_JOB_CMD, command);
	dprintf(D_FULLDEBUG, "%s: '%s'\n", ATTR_JOB_CMD, command.c_str());

	std::string sandboxPath = Starter->jic->jobRemoteIWD();

#ifdef WIN32
	#if 1
	// TODO: make this configurable? and the same on Windows/Linux
	// TODO: make it settable by the job so that we can support Windows containers on Windows?
	std::string innerdir("/test/execute/");
	Starter->SetInnerWorkingDir(innerdir.c_str());
	#else
	const char * outerdir = Starter->GetWorkingDir(false);
	std::string innerdir(outerdir);
	const char * tmp = strstr(outerdir, "\\execute\\");
	if (tmp) {
		innerdir = tmp;
		std::replace(innerdir.begin(), innerdir.end(), '\\', '/');
		Starter->SetInnerWorkingDir(innerdir.c_str());
	}
	#endif
#else
	// TODO: make this work on Linux also
	std::string innerdir = Starter->jic->jobRemoteIWD();
#endif

	bool transferExecutable = true;
	JobAd->LookupBool( ATTR_TRANSFER_EXECUTABLE, transferExecutable );
	if( Starter->jic->usingFileTransfer() && transferExecutable ) {
		std::string old_cmd = command;
		formatstr(command, "./%s", condor_basename(old_cmd.c_str()));
	}

	ArgList args;
	args.SetArgV1SyntaxToCurrentPlatform();
	std::string argsError;
	if( ! args.AppendArgsFromClassAd( JobAd, argsError ) ) {
		dprintf( D_ERROR, "Failed to read job arguments from job ad: '%s'.\n", argsError.c_str() );
		return FALSE;
	}
	//
	// temporary hack to allow condor_submit -i to work better
	// with docker universe
	bool isInteractive = false;
	JobAd->LookupBool("InteractiveJob", isInteractive);	
	if (isInteractive) {
		args.AppendArg("86400");
	}

	Env job_env;
	std::string env_errors;
	if( !Starter->GetJobEnv(JobAd,&job_env, env_errors) ) {
		dprintf( D_ALWAYS, "Aborting DockerProc::StartJob: %s\n", env_errors.c_str());
		return 0;
	}

	// The GlobalJobID is unsuitable by virtue its octothorpes.  This
	// construction is informative, but could be made even less likely
	// to collide if it had a timestamp.
	formatstr( containerName, "HTCJob%d_%d_%s_PID%d",
		Starter->jic->jobCluster(),
		Starter->jic->jobProc(),
		Starter->getMySlotName().c_str(), // note: this can be "" for single slot machines.
		getpid() );

	ClassAd recoveryAd;
	recoveryAd.Assign("DockerContainerName", containerName);
	Starter->WriteRecoveryFile(&recoveryAd);

	int childFDs[3] = { 0, 0, 0 };
	{
	TemporaryPrivSentry sentry(PRIV_USER);
	std::string workingDir = Starter->GetWorkingDir(0);
	//std::string DockerOutputFile = workingDir + "/docker_stdout";
	std::string DockerErrorFile  = workingDir + "/docker_stderror";

	//childFDs[1] = open(DockerOutputFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
	childFDs[2] = open(DockerErrorFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
	}

	  // Ulog the execute event
	Starter->jic->notifyJobPreSpawn();

	CondorError err;
	// DockerAPI::createContainer() returns a PID from daemonCore->Create_Process(), which
	// makes it suitable for passing up into VanillaProc.  This combination
	// will trigger the reaper(s) when the container terminates.
	
	ClassAd *machineAd = Starter->jic->machClassAd();

	std::list<std::string> extras;
	std::string scratchDir = Starter->GetWorkingDir(0);

	// map the scratch dir inside the container
	extras.push_back(scratchDir + ":" + scratchDir);

	// if file xfer is off, also map the iwd
	std::string iwd = Starter->jic->jobRemoteIWD();
	if (iwd != scratchDir) {
		extras.push_back(iwd + ":" + iwd);
	}

	buildExtraVolumes(extras, *machineAd, *JobAd);

	int *affinity_mask = makeCpuAffinityMask(Starter->getMySlotNumber());

	// The following line is for condor_who to parse
	dprintf( D_ALWAYS, "About to exec docker:%s\n", command.c_str());
	int rv = DockerAPI::createContainer( *machineAd, *JobAd,
		containerName, imageID,
		command, args, job_env,
		sandboxPath, innerdir,
		extras, JobPid, childFDs,
		shouldAskForServicePorts, err, affinity_mask);
	if( rv < 0 ) {
		dprintf( D_ERROR, "DockerAPI::createContainer( %s, %s, ... ) failed with return value %d\n", imageID.c_str(), command.c_str(), rv );
		handleFTL(rv);
		return FALSE;
	}
	dprintf( D_FULLDEBUG, "DockerAPI::createContainer() returned pid %d\n", JobPid );
	// The following line is for condor_who to parse
	dprintf( D_ALWAYS, "Create_Process succeeded, pid=%d\n", JobPid);
	// If we manage to start the Docker job, clear the offline state for docker universe
	handleFTL(0);

	free(affinity_mask);
	waitForCreate = true;  // Tell the reaper to run start container on exit
	return TRUE;
}

int execPid;
ReliSock *ns;

int
DockerProc::ExecReaper(int pid, int status) {
	dprintf( D_FULLDEBUG, "DockerProc::ExecReaper() pid is %d with status %d\n", pid, status);
	if (pid == execPid) {
		dprintf(D_ALWAYS, "docker exec pid %d exited\n", pid);
		delete ns;
		return 0;
	} else {
		dprintf(D_ALWAYS, "docker exec unknown pid %d exited\n", pid);
	}
	return 1;
}

bool DockerProc::JobReaper( int pid, int status ) {
	dprintf( D_FULLDEBUG, "DockerProc::JobReaper() pid is %d status is %d wait_for_Create is %d\n", pid, status, waitForCreate);

	if (waitForCreate) {
		// When we get here, the docker create container has exited

		if (status != 0) {
			// Error creating container.  Perhaps invalid image
			std::string message;

			char buf[512];
			buf[0] = '\0';

			{
			TemporaryPrivSentry sentry(PRIV_USER);
			std::string fileName = Starter->GetWorkingDir(0);
			fileName += "/docker_stderror";
			int fd = open(fileName.c_str(), O_RDONLY, 0000);
			if (fd >= 0) {
				int r = read(fd, buf, 511);
				if (r < 0) {
					dprintf(D_ALWAYS, "Cannot read docker error file on docker create container. Errno %d\n", errno);
					buf[0] = '\0';
				} else {
					buf[r] = '\0';
					int buflen = strlen(buf);
					for (int i = 0; i < buflen; i++) {
						if (buf[i] == '\n') buf[i] = ' ';
					}
				}
				close(fd);
			} else {
				dprintf(D_ALWAYS, "Cannot open docker_stderror\n");
			}
			}
			message = buf;
			Starter->jic->holdJob(message.c_str(), CONDOR_HOLD_CODE::InvalidDockerImage, 0);
			{
			TemporaryPrivSentry sentry(PRIV_USER);
			unlink("docker_stderror");
			}
			return VanillaProc::JobReaper( pid, status );
		}

		// When we get here docker create has just succeeded
		waitForCreate = false;
		dprintf(D_FULLDEBUG, "DockerProc::JobReaper docker create (pid %d) exited with status %d\n", pid, status);

		// It would be nice if we could find out what port Docker selected
		// for search service (if any) before we actually started the job,
		// but (understandably) Docker doesn't do that.

	#ifdef WIN32 
		#ifdef COPY_INPUT_SANDBOX
		// copy the input sandbox into the container
		{
			std::string workingDir = Starter->GetWorkingDir(0);
			std::string innerPath = Starter->GetWorkingDir(true);
			StringList opts("-a");

			//TODO: figure out if we need to do this, or to switch to  PRIV_USER
			//TemporaryPrivSentry sentry(PRIV_ROOT);

			int rv = DockerAPI::copyToContainer(workingDir, containerName, innerPath, &opts);
			if (rv < 0) {
				dprintf(D_ERROR, "DockerAPI::copyToContainer( %s, %s, %s ) failed with return value %d\n",
					workingDir.c_str(), containerName.c_str(), innerPath.c_str(), rv);
				return FALSE;
			}
		}
		#endif
	#endif

		// It seems like this should be done _after_ we call start Container().
		Starter->SetJobEnvironmentReady(true);


		CondorError err;

		//
		// Do I/O redirection (includes streaming).
		//

		int childFDs[3] = { -2, -2, -2 };
		{
		TemporaryPrivSentry sentry(PRIV_USER);
		// getStdFile() returns -1 on error.

		if( -1 == (childFDs[0] = openStdFile( SFT_IN, NULL, true, "Input file" )) ) {
			dprintf( D_ERROR, "DockerProc::StartJob(): failed to open stdin.\n" );
			return FALSE;
		}

		if( -1 == (childFDs[1] = openStdFile( SFT_OUT, NULL, true, "Output file" )) ) {

			dprintf( D_ERROR, "DockerProc::StartJob(): failed to open stdout.\n" );
			daemonCore->Close_FD( childFDs[0] );
			return FALSE;
		}
		if( -1 == (childFDs[2] = openStdFile( SFT_ERR, NULL, true, "Error file" )) ) {
			dprintf( D_ERROR, "DockerProc::StartJob(): failed to open stderr.\n" );
			daemonCore->Close_FD( childFDs[0] );
			daemonCore->Close_FD( childFDs[1] );
			return FALSE;
		}
		}

		{
		TemporaryPrivSentry sentry(PRIV_ROOT);
		DockerAPI::startContainer( containerName, JobPid, childFDs, err );
		}
		condor_gettimestamp( job_start_time );
	
		// Start a timer to poll for job usage updates.
		int polling_interval = param_integer("POLLING_INTERVAL",5);
		updateTid = daemonCore->Register_Timer(2,
				polling_interval, (TimerHandlercpp)&DockerProc::getStats, 
					"DockerProc::getStats",this);

		bool ssh_enabled = param_boolean("ENABLE_SSH_TO_JOB",true,true,Starter->jic->machClassAd(),JobAd);
		if( ssh_enabled ) {
			SetupDockerSsh();
		}

		++num_pids; // Used by OsProc::PublishUpdateAd().
		return false; // don't exit
	}


	//
	// This should mean that the container has terminated.
	//
	if( pid == JobPid ) {
		daemonCore->Cancel_Timer(updateTid);

		// Once the container has terminated, I don't care what Docker
		// thinks; nobody's going to be answering the phone.
		std::string containerServiceNames;
		JobAd->LookupString(ATTR_CONTAINER_SERVICE_NAMES, containerServiceNames);
		if(! containerServiceNames.empty()) {
			StringList services(containerServiceNames.c_str());
			services.rewind();
			const char * service = NULL;
			while( NULL != (service = services.next()) ) {
				std::string attrName;
				formatstr( attrName, "%s_%s", service, "HostPort" );
				serviceAd.Insert( attrName, classad::Literal::MakeUndefined() );
			}
		}

		TemporaryPrivSentry sentry(PRIV_ROOT);

		//
		// Even running Docker in attached mode, we have a race condition
		// is exiting when the container exits, not when the docker daemon
		// notices that the container has exited.
		//

		int rv = -1;
		bool running = false;
		ClassAd dockerAd;
		CondorError error;
		// Years of careful research.
		for( int i = 0; i < 20; ++i ) {
			rv = DockerAPI::inspect( containerName, & dockerAd, error );
			if( rv < 0 ) {
				dprintf( D_FULLDEBUG, "Failed to inspect (for removal) container '%s'; sleeping a second (%d already slept) to give Docker a chance to catch up.\n", containerName.c_str(), i );
				sleep( 1 );
				continue;
			}

			if( ! dockerAd.LookupBool( "Running", running ) ) {
				dprintf( D_FULLDEBUG, "Inspection of container '%s' failed to reveal its running state; sleeping a second (%d already slept) to give Docker a chance to catch up.\n", containerName.c_str(), i );
				sleep( 1 );
				continue;
			}

			if( running ) {
				dprintf( D_FULLDEBUG, "Inspection reveals that container '%s' is still running; sleeping a second (%d already slept) to give Docker a chance to catch up.\n", containerName.c_str(), i );
				sleep( 1 );
				continue;
			}

			break;
		}

// FIXME: Move all this shared conditional-checking into a function.

		if( rv < 0 ) {
			dprintf( D_ERROR, "Failed to inspect (for removal) container '%s'.\n", containerName.c_str() );
			EXCEPT("Cannot inspect exited container");
		}

		if( ! dockerAd.LookupBool( "Running", running ) ) {
			dprintf( D_ERROR, "Inspection of container '%s' failed to reveal its running state.\n", containerName.c_str() );
			return VanillaProc::JobReaper( pid, status );
		}

		if( running ) {
			dprintf( D_ERROR, "Inspection reveals that container '%s' is still running.\n", containerName.c_str() );
			return VanillaProc::JobReaper( pid, status );
		}

		// FIXME: Rethink returning a classad.  Having to check for missing
		// attributes blows.

		// TODO: Set status appropriately (as if it were from waitpid()).
		std::string oomkilled;
		if (! dockerAd.LookupString( "OOMKilled", oomkilled)) {
			dprintf( D_ERROR, "Inspection of container '%s' failed to reveal whether it was OOM killed. Assuming it was not.\n", containerName.c_str() );
		}

		if (oomkilled.find("true") == 0) {
			ClassAd *machineAd = Starter->jic->machClassAd();
			int memory;
			machineAd->LookupInteger(ATTR_MEMORY, memory);
			std::string message;
			formatstr(message, "Docker job has gone over memory limit of %d Mb", memory);
			dprintf(D_ALWAYS, "%s, going on hold\n", message.c_str());


			Starter->jic->holdJob(message.c_str(), CONDOR_HOLD_CODE::JobOutOfResources, 0);
			DockerAPI::rm( containerName, error );

			if ( Starter->Hold( ) ) {
				Starter->allJobsDone();
				this->JobExit();
			}

			Starter->ShutdownFast();
			return 0;
		}

			// See if docker could not run the job
			// most likely invalid executable
		std::string dockerError;
		if (! dockerAd.LookupString( "DockerError", dockerError)) {
			dprintf( D_ERROR, "Inspection of container '%s' failed to reveal whether there was an internal docker error.\n", containerName.c_str() );
		}

		if (dockerError.length() > 0) {
			std::string message;
			formatstr(message, "Error running docker job: %s", dockerError.c_str());
			dprintf(D_ALWAYS, "%s, going on hold\n", message.c_str());


			Starter->jic->holdJob(message.c_str(), CONDOR_HOLD_CODE::FailedToCreateProcess, 0);
			DockerAPI::rm( containerName, error );

			if ( Starter->Hold( ) ) {
				Starter->allJobsDone();
				this->JobExit();
			}

			Starter->ShutdownFast();
			return 0;
		}

		int dockerStatus;
		if( ! dockerAd.LookupInteger( "ExitCode", dockerStatus ) ) {
			dprintf( D_ERROR, "Inspection of container '%s' failed to reveal its exit code.\n", containerName.c_str() );
			return VanillaProc::JobReaper( pid, status );
		}
		status = dockerStatus > 128 ? (dockerStatus - 128) : (dockerStatus << 8);
		dprintf( D_FULLDEBUG, "Setting status of Docker job to %d.\n", status );

		// TODO: Record final job usage.

		// We don't have to do any process clean-up, because container.
		// We'll do the disk clean-up after we've transferred files.

		// This helps to make ssh-to-job more plausible.
		return VanillaProc::JobReaper( pid, status );
	}

	return 0;
}

void
DockerProc::SetupDockerSsh() {
#ifdef LINUX
	static bool first_time = true;
	if (first_time) {
		first_time = false;
	} else {
		return;
	}

	// First, create a unix domain socket that we can listen on
	int uds = socket(AF_UNIX, SOCK_STREAM, 0);
	if (uds < 0) {
		dprintf(D_ALWAYS, "Cannot create unix domain socket for docker ssh_to_job\n");
		return;
	}

	// stuff the unix domain socket into a reli sock
	listener.close();
	listener.assignDomainSocket(uds);

	// and bind it to a filename in the scratch directory
	struct sockaddr_un pipe_addr;
	memset(&pipe_addr, 0, sizeof(pipe_addr));
	pipe_addr.sun_family = AF_UNIX;
	unsigned pipe_addr_len;

	std::string workingDir = Starter->GetWorkingDir(0);
	std::string pipeName = workingDir + "/.docker_sock";	

	strncpy(pipe_addr.sun_path, pipeName.c_str(), sizeof(pipe_addr.sun_path)-1);
	pipe_addr_len = SUN_LEN(&pipe_addr);

	{
	TemporaryPrivSentry sentry(PRIV_USER);
	int rc = bind(uds, (struct sockaddr *)&pipe_addr, pipe_addr_len);
	if (rc < 0) {
		dprintf(D_ALWAYS, "Cannot bind unix domain socket at %s for docker ssh_to_job: %d\n", pipeName.c_str(), errno);
		return;
	}
	}

	listen(uds, 50);
	listener._state = Sock::sock_special;
	listener._special_state = ReliSock::relisock_listen;

	// now register this socket so we get called when connected to
	int rc;
	rc = daemonCore->Register_Socket(
		&listener,
		"Docker sshd listener",
		(SocketHandlercpp)&DockerProc::AcceptSSHClient,
		"DockerProc::AcceptSSHClient",
		this);
	ASSERT( rc >= 0 );

	// and a reaper 
	execReaperId = daemonCore->Register_Reaper("ExecReaper", (ReaperHandlercpp)&DockerProc::ExecReaper,
		"ExecReaper", this);
#else
	// Shut the compiler up
	(void)execReaperId;
#endif
	return;
}


int
DockerProc::AcceptSSHClient(Stream *stream) {
#ifdef LINUX
	int fds[3];
	ns = ((ReliSock*)stream)->accept();

	dprintf(D_ALWAYS, "Accepted new connection from ssh client for docker job\n");
	fds[0] = fdpass_recv(ns->get_file_desc());
	fds[1] = fdpass_recv(ns->get_file_desc());
	fds[2] = fdpass_recv(ns->get_file_desc());

	ArgList args;
	args.AppendArg("-i");

	Env env;
	std::string env_errors;
	if( !Starter->GetJobEnv(JobAd,&env, env_errors) ) {
		dprintf( D_ALWAYS, "Aborting DockerProc::exec: %s\n", env_errors.c_str());
		return 0;
	}

	int rc;
	{
	TemporaryPrivSentry sentry(PRIV_ROOT);

	rc = DockerAPI::execInContainer(
	 containerName, "/bin/bash", args, env,fds,execReaperId,execPid);
	}

	dprintf(D_ALWAYS, "docker exec returned %d for pid %d\n", rc, execPid);

#else
	// Shut the compiler up
	(void)stream;
#endif
	return KEEP_STREAM;
}

//
// JobExit() is called after file transfer.
//
bool DockerProc::JobExit() {
	dprintf( D_ALWAYS, "DockerProc::JobExit() container '%s'\n", containerName.c_str() );

	{
	TemporaryPrivSentry sentry(PRIV_ROOT);
	ClassAd dockerAd;
	CondorError error;
	int rv = DockerAPI::inspect( containerName, & dockerAd, error );
	if( rv < 0 ) {
		dprintf( D_ERROR, "Failed to inspect (for removal) container '%s'.\n", containerName.c_str() );
		return VanillaProc::JobExit();
	}

	bool running;
	if( ! dockerAd.LookupBool( "Running", running ) ) {
		dprintf( D_ERROR, "Inspection of container '%s' failed to reveal its running state.\n", containerName.c_str() );
		return VanillaProc::JobExit();
	}
	if( running ) {
		dprintf( D_ERROR, "Inspection reveals that container '%s' is still running.\n", containerName.c_str() );
		return VanillaProc::JobExit();
	}

	rv = DockerAPI::rm( containerName, error );
	if( rv < 0 ) {
		dprintf( D_ERROR, "Failed to remove container '%s'.\n", containerName.c_str() );
	}
	}

	return VanillaProc::JobExit();
}

void DockerProc::Suspend() {
	dprintf( D_ALWAYS, "DockerProc::Suspend() container '%s'\n", containerName.c_str() );
	int rv = 0;

	{
		TemporaryPrivSentry sentry(PRIV_ROOT);
		CondorError error;
		rv = DockerAPI::pause( containerName, error );
	}
	TemporaryPrivSentry sentry(PRIV_ROOT);
	if( rv < 0 ) {
		dprintf( D_ERROR, "Failed to suspend container '%s'.\n", containerName.c_str() );
	}

	is_suspended = true;
}


void DockerProc::Continue() {
	dprintf( D_ALWAYS, "DockerProc::Continue() container '%s'\n", containerName.c_str() );
	int rv = 0;	

	if( is_suspended ) {
		{
			TemporaryPrivSentry sentry(PRIV_ROOT);
			CondorError error;
			rv = DockerAPI::unpause( containerName, error );
		}
		if( rv < 0 ) {
			dprintf( D_ERROR, "Failed to unpause container '%s'.\n", containerName.c_str() );
		}


		is_suspended = false;
	}
}

//
// Setting requested_exit allows OsProc::JobExit() to handle telling the
// user why the job exited.
//

bool DockerProc::Remove() {
	dprintf( D_ALWAYS, "DockerProc::Remove() container '%s'\n", containerName.c_str() );

	if( is_suspended ) { Continue(); }
	requested_exit = true;

	CondorError err;
	TemporaryPrivSentry sentry(PRIV_ROOT);
	DockerAPI::kill( containerName, rm_kill_sig, err );

	// Do NOT send any signals to the waiting process.  It should only
	// react when the container does.

	// If rm_kill_sig is not SIGKILL, the process may linger.  Returning
	// false indicates that shutdown is pending.
	return false;
}


bool DockerProc::Hold() {
	dprintf( D_ALWAYS, "DockerProc::Hold() container '%s'\n", containerName.c_str() );

	if( is_suspended ) { Continue(); }
	requested_exit = true;

	CondorError err;
	TemporaryPrivSentry sentry(PRIV_ROOT);
	DockerAPI::kill( containerName, hold_kill_sig, err );

	// Do NOT send any signals to the waiting process.  It should only
	// react when the container does.

	// If hold_kill_sig is not SIGKILL, the process may linger.  Returning
	// false indicates that shutdown is pending.
	return false;
}


bool DockerProc::ShutdownGraceful() {
	dprintf( D_ALWAYS, "DockerProc::ShutdownGraceful() container '%s'\n", containerName.c_str() );

	if( containerName.empty() ) {
		// We haven't started a Docker yet, probably because we're still
		// doing file transfer.  Since we're all done, just return true;
		// the FileTransfer object will clean itself up.
		return true;
	}

	if( is_suspended ) { Continue(); }
	requested_exit = true;

	CondorError err;
	TemporaryPrivSentry sentry(PRIV_ROOT);
	if( findRmKillSig(JobAd) != -1 ) {
		DockerAPI::kill( containerName, rm_kill_sig, err );
	} else {
		DockerAPI::kill( containerName, soft_kill_sig, err );
	}

	// If the signal was not SIGKILL, the process may linger.  Returning
	// false indicates that shutdown is pending.
	return false;
}


bool DockerProc::ShutdownFast() {
	dprintf( D_ALWAYS, "DockerProc::ShutdownFast() container '%s'\n", containerName.c_str() );

	if( containerName.empty() ) {
		// We haven't started a Docker yet, probably because we're still
		// doing file transfer.  Since we're all done, just return true;
		// the FileTransfer object will clean itself up.
		return true;
	}

	// There's no point unpausing the container (and possibly swapping
	// it all back in again) if we're just going to be sending it a SIGKILL,
	// so don't bother to Continue() the process if it's been suspended.
	requested_exit = true;

	CondorError err;
	TemporaryPrivSentry sentry(PRIV_ROOT);
	DockerAPI::kill( containerName, SIGKILL, err );

	// Based on the other comments, you'd expect this to return true.
	// It could, but it's simpler to just to let the usual routines
	// handle the job clean-up than to duplicate them all here.
	return false;
}


void
DockerProc::getStats( int /* timerID */ ) {
	if( shouldAskForServicePorts ) {
		if( DockerAPI::getServicePorts( containerName, * JobAd, serviceAd ) == 0 ) {
			shouldAskForServicePorts = false;

			// Append serviceAd to the sandbox's copy of the job ad.
			std::string jobAdFileName;
			formatstr( jobAdFileName, "%s/.job.ad", Starter->GetWorkingDir(0) );
			{
				TemporaryPrivSentry sentry(PRIV_ROOT);
				// ... sigh ...
				ToE::writeTag( & serviceAd, jobAdFileName );
			}
		}
	}
	DockerAPI::stats(containerName, memUsage, netIn, netOut, userCpu, sysCpu);

	// Since we typically will poll the Docker daemon for memory usage
	// at a higher rate than we update the shadow, keep the maximum observed
	// memory usage here in the starter (and use max_memUsage to update the shadow).
	if (memUsage > max_memUsage) {
		max_memUsage = memUsage;
	}

	return;
}

bool DockerProc::PublishUpdateAd( ClassAd * ad ) {
	dprintf( D_FULLDEBUG, "DockerProc::PublishUpdateAd() container '%s'\n", containerName.c_str() );

	// Copy the service-to-port map into the update ad.
	ad->Update( serviceAd );

	//
	// If we want to use the existing reporting code (probably a good
	// idea), we'll need to make sure that m_proc_exited, is_checkpointed,
	// is_suspended, num_pids, and dumped_core are set for OsProc; and that
	// job_start_time, job_exit_time, and exit_status are set.
	//
	// We set is_suspended and num_pids already, except for the TODO above.
	// DockerProc::JobReaper() already sets m_proc_exited, exit_code, and
	// dumped_core (indirectly, via OsProc::JobReaper()).
	//
	// We will need to set is_checkpointed appropriately when we support it.
	//
	// TODO: We could approximate job_start_time and job_exit_time internally,
	// or set them during our status polling.
	//
	
	if (max_memUsage > 0) {
		// Set RSS, Memory and ImageSize to same values, best we have
		ad->Assign(ATTR_RESIDENT_SET_SIZE, int(max_memUsage / 1024));
		ad->Assign(ATTR_MEMORY_USAGE, int(max_memUsage / (1024 * 1024)));
		ad->Assign(ATTR_IMAGE_SIZE, int(max_memUsage / (1024 * 1024)));
		ad->Assign(ATTR_NETWORK_IN, double(netIn) / (1000 * 1000));
		ad->Assign(ATTR_NETWORK_OUT, double(netOut) / (1000 * 1000));
		ad->Assign(ATTR_JOB_REMOTE_USER_CPU, (int) (userCpu / (1000l * 1000l * 1000l)));
		ad->Assign(ATTR_JOB_REMOTE_SYS_CPU, (int) (sysCpu / (1000l * 1000l * 1000l)));
	}
	return OsProc::PublishUpdateAd( ad );
}


// TODO: Implement.
void DockerProc::PublishToEnv( Env * /* env */ ) {
	dprintf( D_FULLDEBUG, "DockerProc::PublishToEnv()\n" );
	return;
}


bool DockerProc::Detect() {
	dprintf( D_FULLDEBUG, "DockerProc::Detect()\n" );

	//
	// To turn off Docker, unset DOCKER.  DockerAPI::detect() will fail
	// but not complain to the log (unless D_FULLDEBUG) if so.
	//

	CondorError err;
	bool hasDocker = DockerAPI::detect( err ) == 0;

	if (hasDocker) {
		int r  = DockerAPI::testImageRuns(err);
		if (r == 0) {
			hasDocker = true;
		} else {
			hasDocker = false;
		}
	}

	return hasDocker;
}

bool DockerProc::Version( std::string & version ) {
	dprintf( D_FULLDEBUG, "DockerProc::Version()\n" );

	//
	// To turn off Docker, unset DOCKER.  DockerAPI::version() will fail
	// but not complain to the log (unless D_FULLDEBUG) if so.
	//

	CondorError err;
	bool foundVersion = DockerAPI::version( version, err ) == 0;
	if (foundVersion) {
		dprintf( D_ALWAYS, "DockerProc::Version() found version '%s'\n", version.c_str() );
	}

	return foundVersion;
}
void 
DockerProc::restartCheckpointedJob() {
	TemporaryPrivSentry sentry(PRIV_ROOT);
	CondorError error;
	int rv = DockerAPI::rm( containerName, error );
	if( rv < 0 ) {
		dprintf( D_ERROR, "Failed to remove container '%s' after checkpoint exit.\n", containerName.c_str() );
		// Will fail later when we try to restart if it still exists.  If it doesn't :shrug: all good!
	}

	VanillaProc::restartCheckpointedJob();
}

// Generate a list of strings that are suitable arguments to
// docker run --volume
static void buildExtraVolumes(std::list<std::string> &extras, ClassAd &machAd, ClassAd &jobAd) {
	// Bind mount items from MOUNT_UNDER_SCRATCH into working directory
	std::string scratchNames;
	if (!param_eval_string(scratchNames, "MOUNT_UNDER_SCRATCH", "", &jobAd)) {
		// If not an expression, maybe a literal
		param(scratchNames, "MOUNT_UNDER_SCRATCH");
	} 

#ifdef DOCKER_ALLOW_RUN_AS_ROOT
		// If docker is allowing the user to be root, don't mount anything
		// so that we can't create rootly files in shared places.
	if (param_boolean("DOCKER_RUN_AS_ROOT", false)) {
		return;
	}
#endif

	if (scratchNames.length() > 0) {
		std::string workingDir = Starter->GetWorkingDir(0);
		StringList sl(scratchNames.c_str());
		sl.rewind();
		char *scratchName = 0;
			// Foreach scratch name...
		while ( (scratchName=sl.next()) ) {
			std::string hostdirbuf;
			const char * hostDir = dirscat(workingDir.c_str(), scratchName, hostdirbuf);
			std::string volumePath;
			volumePath.append(hostDir).append(":").append(scratchName);
			
			mode_t old_mask = umask(000);
			if (mkdir_and_parents_if_needed( hostDir, 01777, PRIV_USER )) {
				extras.push_back(volumePath);
				dprintf(D_ALWAYS, "Adding %s as a docker volume to mount under scratch\n", volumePath.c_str());
			} else {
				dprintf(D_ALWAYS, "Failed to create scratch directory %s\n", hostDir);
			}
			umask(old_mask);
		}
	}

	// These are the ones the administrator wants unconditionally mounted
	char *volumeNames = param("DOCKER_MOUNT_VOLUMES");
	if (!volumeNames) {
		// Nothing extra to mount, give up
		return;
	}

	StringList vl(volumeNames);
	vl.rewind();
	char *volumeName = 0;
		// Foreach volume name...
	while ( (volumeName=vl.next()) ) {
		std::string paramName("DOCKER_VOLUME_DIR_");
		paramName += volumeName;
		char *volumePath = param(paramName.c_str());
		if (volumePath) {
			if (strchr(volumePath, ':') == 0) {
				// Must have a colon.  If none, assume we meant
				// source:source
				char *volumePath2 = (char *)malloc(1 + 2 * strlen(volumePath));
				strcpy(volumePath2, volumePath);
				strcat(volumePath2, ":");
				strcat(volumePath2, volumePath);
				free(volumePath);
				volumePath = volumePath2;
			}

			// If there's a DOCKER_VOLUME_DIR_XXX_MOUNT_IF
			// param, and it evals to true, add it to the list
			// if there's no param, still add it
			// If the param exists, but doesn't eval to true, don't
			std::string paramNameConst = paramName + "_MOUNT_IF";
			char *mountIfValue = param(paramNameConst.c_str());

			if (mountIfValue == NULL) {
				// there's no MOUNT_IF, assume true
				extras.push_back(volumePath);
				dprintf(D_ALWAYS, "Adding %s as a docker volume to mount\n", volumePath);
			} else if (param_boolean(paramNameConst.c_str(), false /*deflt*/,
				false /*do_log*/, &machAd, &jobAd)) {

				// There is a MOUNT_IF, and it is true
				extras.push_back(volumePath);
				dprintf(D_ALWAYS, "Adding %s as a docker volume to mount\n", volumePath);
			} else {
				// There is a MOUNT_IF, and it is false/undef
				dprintf(D_ALWAYS, "Not adding %s as a docker volume to mount -- ...MOUNT_IF evaled to false.\n", volumePath);
			}
			free(volumePath);
			free(mountIfValue);
		} else {
			dprintf(D_ALWAYS, "WARNING: DOCKER_VOLUME_DIR_%s is missing in config file.\n", volumeName);
		}
	}
}
