Merge pull request #14116 from stevekuznetsov/skuznets/cleanup_all

Merged by openshift-bot

Author: OpenShift Bot

hack/common.sh

@@ -210,11 +210,6 @@ os::build::internal::build_binaries() {
       fi
     done
 
-    # Temporarily enable swap for the duration of the build until we move
-    # to Go 1.7
-    os::build::enable_swap
-    trap "os::build::disable_swap" EXIT
-
     local host_platform=$(os::build::host_platform)
     local platform
     for platform in "${platforms[@]+"${platforms[@]}"}"; do
@@ -785,87 +780,6 @@ function os::build::image() {
 }
 readonly -f os::build::image
 
-# os::build::enable_swap attempts to enable swap for the system if a) this is Linux and b)
-# the amount of physical memory is less than 10GB. This is a stopgap until we have
-# better control over memory use in Go 1.7+.
-function os::build::enable_swap() {
-  # if we aren't on linux or have more than 9GB of memory
-  if [[ -n "${OS_BUILD_SWAP_DISABLE-}" || "$(go env GOHOSTOS)" != "linux" || "$( os::build::physmem_gb )" -gt 9 ]]; then
-    return
-  fi
-  # if we don't have the swapon command available
-  if ! swapon &>/dev/null; then
-    return
-  fi
-  # if swap is already on
-  if [[ "$( swapon --show --noheadings | wc -l )" -ne 0 ]]; then
-    return
-  fi
-  echo "++ temporarily enabling swap space to assist in building in limited memory - use OS_BUILD_SWAP_DISABLE=1 to bypass"
-
-  (
-    set -e
-    sudo cp /etc/fstab /origin-backupfstab
-    sudo dd if=/dev/zero of=/origin-swapfile bs=1M count=${OS_BUILD_SWAP_SIZE:-2048}
-    sudo chmod 600 /origin-swapfile
-    sudo mkswap /origin-swapfile
-    sudo swapon /origin-swapfile
-    sudo /bin/sh -c 'echo "/origin-swapfile none swap defaults 0 0" >> /etc/fstab'
-  ) &>/dev/null || true
-}
-readonly -f os::build::enable_swap
-
-# os::build::disable_swap undoes the effects of os::build::enable_swap
-function os::build::disable_swap() {
-  # if we aren't on linux or have more than 9GB of memory
-  if [[ -n "${OS_BUILD_SWAP_DISABLE-}" || "$(go env GOHOSTOS)" != "linux" ]]; then
-    return
-  fi
-  # if we previously set up a swapfile
-  if [[ ! -f /origin-swapfile ]]; then
-    return
-  fi
-  (
-    set +e
-    sudo swapoff /origin-swapfile
-    sudo cp /origin-backupfstab /etc/fstab
-    sudo rm -f /origin-backupfstab /origin-swapfile
-  ) || true
-}
-readonly -f os::build::disable_swap
-
-# os::build::physmem_gb returns the approximate gigabytes of memory on the system
-# This is copied verbatim from kube for the purposes of detecting how much available
-# memory we have for building.
-function os::build::physmem_gb() {
-  local mem
-
-  # Linux kernel version >=3.14, in kb
-  if mem=$(grep MemAvailable /proc/meminfo | awk '{ print $2 }'); then
-    echo $(( ${mem} / 1048576 ))
-    return
-  fi
-
-  # Linux, in kb
-  if mem=$(grep MemTotal /proc/meminfo | awk '{ print $2 }'); then
-    echo $(( ${mem} / 1048576 ))
-    return
-  fi
-
-  # OS X, in bytes. Note that get_physmem, as used, should only ever
-  # run in a Linux container (because it's only used in the multiple
-  # platform case, which is a Dockerized build), but this is provided
-  # for completeness.
-  if mem=$(sysctl -n hw.memsize 2>/dev/null); then
-    echo $(( ${mem} / 1073741824 ))
-    return
-  fi
-
-  # If we can't infer it, just give up and assume a low memory system
-  echo 1
-}
-readonly -f os::build::physmem_gb
-
 # os::build::require_clean_tree exits if the current Git tree is not clean.
 function os::build::require_clean_tree() {
   if ! git diff-index --quiet HEAD -- || test $(git ls-files --exclude-standard --others | wc -l) != 0; then

hack/lib/cleanup.sh

@@ -3,18 +3,79 @@
 # This library holds functions that are used to clean up local
 # system state after other scripts have run.
 
+# os::cleanup::all will clean up all of the processes and data that
+# a script leaves around after running. All of the sub-tasks called
+# from this function should gracefully handle when they do not need
+# to do anything.
+#
+# Globals:
+#  - ARTIFACT_DIR
+#  - SKIP_TEARDOWN
+#  - SKIP_IMAGE_CLEANUP
+# Arguments:
+#  1 - return code of the script
+# Returns:
+#  None
+function os::cleanup::all() {
+	local return_code="$1"
+
+	# All of our cleanup is best-effort, so we do not care
+	# if any specific step fails.
+	set +o errexit
+
+	os::test::junit::generate_report
+
+	os::log::info "[CLEANUP] Beginning cleanup routines..."
+	os::cleanup::dump_events
+	os::cleanup::dump_etcd
+	os::cleanup::dump_container_logs
+	os::cleanup::dump_pprof_output
+	os::cleanup::find_cache_alterations
+	os::cleanup::truncate_large_logs
+
+	if [[ -z "${SKIP_TEARDOWN:-}" ]]; then
+		os::cleanup::containers
+		os::cleanup::processes
+		os::cleanup::prune_etcd
+	fi
+	os::util::describe_return_code "${return_code}"
+}
+readonly -f os::cleanup::all
+
 # os::cleanup::dump_etcd dumps the full contents of etcd to a file.
 #
 # Globals:
-#  ARTIFACT_DIR
+#  - ARTIFACT_DIR
+#  - API_SCHEME
+#  - API_HOST
+#  - ETCD_PORT
 # Arguments:
 #  None
 # Returns:
 #  None
 function os::cleanup::dump_etcd() {
-	os::log::info "Dumping etcd contents to ${ARTIFACT_DIR}/etcd_dump.json"
-	os::util::curl_etcd "/v2/keys/?recursive=true" > "${ARTIFACT_DIR}/etcd_dump.json"
+	if [[ -n "${API_SCHEME:-}" && -n "${API_HOST:-}" && -n "${ETCD_PORT:-}" ]]; then
+		os::log::info "[CLEANUP] Dumping etcd contents to $( os::util::repository_relative_path "${ARTIFACT_DIR}/etcd_dump.json" )"
+		os::util::curl_etcd "/v2/keys/?recursive=true" > "${ARTIFACT_DIR}/etcd_dump.json"
+	fi
 }
+readonly -f os::cleanup::dump_etcd
+
+# os::cleanup::prune_etcd removes the etcd data store from disk.
+#
+# Globals:
+#  ARTIFACT_DIR
+# Arguments:
+#  None
+# Returns:
+#  None
+function os::cleanup::prune_etcd() {
+	if [[ -n "${ETCD_DATA_DIR:-}" ]]; then
+		os::log::info "[CLEANUP] Pruning etcd data directory"
+		${USE_SUDO:+sudo} rm -rf "${ETCD_DATA_DIR}"
+	fi
+}
+readonly -f os::cleanup::prune_etcd
 
 # os::cleanup::containers operates on our containers to stop the containers
 # and optionally remove the containers and any volumes they had attached.
@@ -27,11 +88,11 @@ function os::cleanup::dump_etcd() {
 #  None
 function os::cleanup::containers() {
 	if ! os::util::find::system_binary docker >/dev/null 2>&1; then
-		os::log::warning "No \`docker\` binary found, skipping container cleanup."
+		os::log::warning "[CLEANUP] No \`docker\` binary found, skipping container cleanup."
 		return
 	fi
 
-	os::log::info "Stopping docker containers"
+	os::log::info "[CLEANUP] Stopping docker containers"
 	for id in $( os::cleanup::internal::list_our_containers ); do
 		os::log::debug "Stopping ${id}"
 		docker stop "${id}" >/dev/null
@@ -41,7 +102,7 @@ function os::cleanup::containers() {
 		return
 	fi
 
-	os::log::info "Removing docker containers"
+	os::log::info "[CLEANUP] Removing docker containers"
 	for id in $( os::cleanup::internal::list_our_containers ); do
 		os::log::debug "Removing ${id}"
 		docker stop "${id}" >/dev/null
@@ -60,14 +121,14 @@ readonly -f os::cleanup::containers
 #  None
 function os::cleanup::dump_container_logs() {
 	if ! os::util::find::system_binary docker >/dev/null 2>&1; then
-		os::log::warning "No \`docker\` binary found, skipping container cleanup."
+		os::log::warning "[CLEANUP] No \`docker\` binary found, skipping container log harvest."
 		return
 	fi
 
 	local container_log_dir="${LOG_DIR}/containers"
 	mkdir -p "${container_log_dir}"
 
-	os::log::info "Dumping container logs to ${container_log_dir}"
+	os::log::info "[CLEANUP] Dumping container logs to $( os::util::repository_relative_path "${container_log_dir}" )"
 	for id in $( os::cleanup::internal::list_our_containers ); do
 		local name; name="$( docker inspect --format '{{ .Name }}' "${id}" )"
 		os::log::debug "Dumping logs for ${id} to ${name}.log"
@@ -138,6 +199,7 @@ readonly -f os::cleanup::internal::list_containers
 # Returns:
 #  None
 function os::cleanup::tmpdir() {
+	os::log::info "[CLEANUP] Cleaning up temporary directories"
 	# ensure that the directories are clean
 	if os::util::find::system_binary "findmnt" &>/dev/null; then
 		for target in $( ${USE_SUDO:+sudo} findmnt --output TARGET --list ); do
@@ -163,11 +225,87 @@ readonly -f os::cleanup::tmpdir
 # Returns:
 #  None
 function os::cleanup::dump_events() {
-	os::log::info "Dumping cluster events to ${ARTIFACT_DIR}/events.txt"
+	os::log::info "[CLEANUP] Dumping cluster events to $( os::util::repository_relative_path "${ARTIFACT_DIR}/events.txt" )"
 	local kubeconfig
-	if [[ -n "${ADMIN_KUBECONFIG}" ]]; then
+	if [[ -n "${ADMIN_KUBECONFIG:-}" ]]; then
 		kubeconfig="--config=${ADMIN_KUBECONFIG}"
 	fi
-	oc get events --all-namespaces ${kubeconfig} > "${ARTIFACT_DIR}/events.txt"
+	oc get events --all-namespaces ${kubeconfig:-} > "${ARTIFACT_DIR}/events.txt" 2>&1
 }
 readonly -f os::cleanup::dump_events
+
+# os::cleanup::find_cache_alterations ulls information out of the server
+# log so that we can get failure management in jenkins to highlight it
+# and really have it smack people in their logs. This is a severe
+# correctness problem.
+#
+# Globals:
+#  - LOG_DIR
+# Arguments:
+#  None
+# Returns:
+#  None
+function os::cleanup::find_cache_alterations() {
+	grep -ra5 "CACHE.*ALTERED" "${LOG_DIR}" || true
+}
+readonly -f os::cleanup::find_cache_alterations
+
+# os::cleanup::dump_pprof_output dumps profiling output for the
+# `openshift` binary
+#
+# Globals:
+#  - LOG_DIR
+# Arguments:
+#  None
+# Returns:
+#  None
+function os::cleanup::dump_pprof_output() {
+	if go tool -n pprof >/dev/null 2>&1 && [[ -s cpu.pprof ]]; then
+		os::log::info "[CLEANUP] \`pprof\` output logged to $( os::util::repository_relative_path "${LOG_DIR}/pprof.out" )"
+		go tool pprof -text "./_output/local/bin/$(os::util::host_platform)/openshift" cpu.pprof >"${LOG_DIR}/pprof.out" 2>&1
+	fi
+}
+readonly -f os::cleanup::dump_pprof_output
+
+# os::cleanup::truncate_large_logs truncates very large files under
+# $LOG_DIR and $ARTIFACT_DIR so we do not upload them to cloud storage
+# after CI runs.
+#
+# Globals:
+#  - LOG_DIR
+#  - ARTIFACT_DIR
+# Arguments:
+#  None
+# Returns:
+#  None
+function os::cleanup::truncate_large_logs() {
+	local max_file_size="100M"
+	os::log::info "[CLEANUP] Truncating log files over ${max_file_size}"
+	for file in $(find "${ARTIFACT_DIR}" "${LOG_DIR}" -type f -name '*.log' \( -size +${max_file_size} \)); do
+		mv "${file}" "${file}.tmp"
+		echo "LOGFILE TOO LONG ($(du -h "${file}.tmp")), PREVIOUS BYTES TRUNCATED. LAST ${max_file_size} OF LOGFILE:" > "${file}"
+		tail -c ${max_file_size} "${file}.tmp" >> "${file}"
+		rm "${file}.tmp"
+	done
+}
+readonly -f os::cleanup::truncate_large_logs
+
+# os::cleanup::processes kills all processes created by the test
+# script.
+#
+# Globals:
+#  None
+# Arguments:
+#  None
+# Returns:
+#  None
+function os::cleanup::processes() {
+	os::log::info "[CLEANUP] Killing child processes"
+	for job in $( jobs -pr ); do
+		for child in $( pgrep -P "${job}" ); do
+			${USE_SUDO:+sudo} kill "${child}" &> /dev/null
+		done
+		${USE_SUDO:+sudo} kill "${job}" &> /dev/null
+	done
+}
+readonly -f os::cleanup::processes

hack/lib/init.sh

@@ -9,6 +9,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+OS_SCRIPT_START_TIME="$( date +%s )"; export OS_SCRIPT_START_TIME
+
 # os::util::absolute_path returns the absolute path to the directory provided
 function os::util::absolute_path() {
 	local relative_path="$1"