#!/bin/sh # Ubuntu-only, variable-driven k3s + Calico + Liqo bootstrap with API integration 12 # Clean logging, robust error handling, TLS insecure by default. # Uses POST /v1/kubernetes-clusters/{id}:generatePeeringConfig → extracts YAML from .config. ############################################################################### # USER VARIABLES # ############################################################################### # --- API / Auth --- API_BASE_URL="${API_BASE_URL:-https://192.168.1.8:20000}" API_TIMEOUT="${API_TIMEOUT:-20}" API_TLS_INSECURE="${API_TLS_INSECURE:-true}" # default: skip TLS verify; set "false" to enforce API_CA_CERT_PATH="${API_CA_CERT_PATH:-}" # if set and exists, use --cacert instead of -k # --- k3s --- K3S_INSTALL_SCRIPT_URL="${K3S_INSTALL_SCRIPT_URL:-https://get.k3s.io}" K3S_KUBECONFIG_MODE="${K3S_KUBECONFIG_MODE:-644}" K3S_CLUSTER_CIDR="${K3S_CLUSTER_CIDR:-192.168.0.0/16}" K3S_FLANNEL_BACKEND="${K3S_FLANNEL_BACKEND:-none}" K3S_INSTALL_DISABLES="${K3S_INSTALL_DISABLES:---disable-network-policy --disable=traefik}" K3S_INSTALL_CHANNEL="${K3S_INSTALL_CHANNEL:-}" KUBECONFIG_PATH="${KUBECONFIG_PATH:-/etc/rancher/k3s/k3s.yaml}" # --- Calico --- CALICO_VERSION="${CALICO_VERSION:-v3.30.2}" CALICO_OP_CRDS_URL="${CALICO_OP_CRDS_URL:-https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/operator-crds.yaml}" CALICO_OPERATOR_URL="${CALICO_OPERATOR_URL:-https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/tigera-operator.yaml}" CALICO_CUSTOM_RES_URL="${CALICO_CUSTOM_RES_URL:-https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/custom-resources.yaml}" CALICO_ROLLOUT_TIMEOUT="${CALICO_ROLLOUT_TIMEOUT:-10m}" # --- Liqo --- LIQO_VERSION="${LIQO_VERSION:-v1.0.1}" LIQO_ARCH_OS="${LIQO_ARCH_OS:-linux-amd64}" LIQOCTL_TGZ_URL="${LIQOCTL_TGZ_URL:-https://github.com/liqotech/liqo/releases/download/${LIQO_VERSION}/liqoctl-${LIQO_ARCH_OS}.tar.gz}" LIQOCTL_PATH="${LIQOCTL_PATH:-/usr/local/bin/liqoctl}" LIQO_TIMEOUT="${LIQO_TIMEOUT:-10m}" LIQO_GW_SERVICE_TYPE="${LIQO_GW_SERVICE_TYPE:-NodePort}" LIQO_GW_SERVER_SERVICE_PORT="${LIQO_GW_SERVER_SERVICE_PORT:-}" # env override > API LIQO_POD_CIDR="${LIQO_POD_CIDR:-$K3S_CLUSTER_CIDR}" # use the same CIDR by default # --- Namespaces & Offloading --- # Space-separated list (POSIX-sh friendly) of namespaces to ensure & offload. NAMESPACES="${NAMESPACES:-operator monitoring}" # Fixed logical name of your remote cluster used in the selector: NODERINGS_CLUSTER_NAME="${NODERINGS_CLUSTER_NAME:-noderings}" # Namespace mapping strategy for liqoctl offload: LIQO_OFFLOAD_STRATEGY="${LIQO_OFFLOAD_STRATEGY:-SelectedName}" # --- Behavior / Logging --- DRY_RUN="${DRY_RUN:-false}" VERBOSE="${VERBOSE:-true}" # log to console as well as file DEBUG="${DEBUG:-false}" # set true to enable shell -x (rarely needed) KEEP_TMP_ON_ERROR="${KEEP_TMP_ON_ERROR:-false}" LOG_DIR="${LOG_DIR:-/var/log/nrings}" LOG_BASENAME="nrings-install-$(date +%Y%m%d%H%M%S).log" LOG_FILE="${LOG_FILE:-$LOG_DIR/$LOG_BASENAME}" LOG_TAIL_LINES="${LOG_TAIL_LINES:-200}" ############################################################################### # BOILERPLATE # ############################################################################### set -eu TMPDIR="$(mktemp -d 2>/dev/null || mktemp -d -t nrings.XXXXXX)" CPATCH="$TMPDIR/calico-patch-k3s.yaml" KPEERCFG_RAW="$TMPDIR/peering-config.raw" KPEERCFG="$TMPDIR/peering-kubeconfig.yaml" API_CLUSTER_JSON="$TMPDIR/cluster.json" STEP_NUM=0 _cleanup_needed="yes" ts() { date +"%Y-%m-%dT%H:%M:%S%z"; } say() { printf '%s %s\n' "$(ts)" "$*"; } # Redact secrets (JWTs / Authorization headers) from any text piped through redact_secrets() { sed -E ' # Authorization: Bearer → Authorization: Bearer ***REDACTED*** s/(Authorization:[[:space:]]*Bearer)([[:space:]]+)(["'\'']?)[^"'\''[:space:]]+/\1\2\3***REDACTED***/Ig; # bare "Bearer " s/([Bb]earer)([[:space:]]+)[^"'\''[:space:]]+/\1\2***REDACTED***/g; # Generic JWT-like triple-segment tokens anywhere s/[A-Za-z0-9_-]{10,}\.[A-Za-z0-9._-]{10,}\.[A-Za-z0-9._-]{10,}/***REDACTED***/g ' } to_log() { mkdir -p "$LOG_DIR" 2>/dev/null || true # keep logs machine-readable; redact before writing printf '%s %s\n' "$(ts)" "$(printf '%s' "$*" | redact_secrets)" >>"$LOG_FILE" 2>/dev/null || true } section() { STEP_NUM=$((STEP_NUM+1)) echo "" say "===== [STEP $STEP_NUM] $* =====" to_log "===== [STEP $STEP_NUM] $* =====" } die() { say "FATAL: $*"; to_log "FATAL: $*" if [ -f "$LOG_FILE" ]; then say "---- LOG TAIL ($LOG_TAIL_LINES) ----" tail -n "$LOG_TAIL_LINES" "$LOG_FILE" || true fi if [ "$KEEP_TMP_ON_ERROR" = "true" ]; then _cleanup_needed="no"; say "Keeping temp dir for inspection: $TMPDIR"; fi exit 1 } cleanup() { if [ "$_cleanup_needed" = "yes" ]; then [ -f "$KPEERCFG" ] && rm -f "$KPEERCFG" [ -d "$TMPDIR" ] && rm -rf "$TMPDIR" fi } trap cleanup INT TERM EXIT is_root() { [ "$(id -u)" -eq 0 ]; } need_cmd() { command -v "$1" >/dev/null 2>&1; } # Accept TOKEN in any of these forms: # - raw JWT # - "Bearer " # - "Authorization: Bearer " auth_header() { case "$TOKEN" in "Authorization: Bearer "*) printf '%s' "$TOKEN" ;; "Bearer "*) printf 'Authorization: %s' "$TOKEN" ;; *) printf 'Authorization: Bearer %s' "$TOKEN" ;; esac } curl_tls_flag() { if [ -n "${API_CA_CERT_PATH:-}" ] && [ -f "$API_CA_CERT_PATH" ]; then printf -- "--cacert '%s'" "$API_CA_CERT_PATH"; return fi if [ "${API_TLS_INSECURE:-true}" = "true" ]; then printf -- "-k"; fi } # Run a command, log start/end, capture output, measure duration, summarize on failure. run() { cmd="$*"; start="$(date +%s)" safe_cmd="$(printf %s "$cmd" | redact_secrets)" say "[RUN] $safe_cmd"; to_log "[RUN] $safe_cmd" [ "$DRY_RUN" = "true" ] && { say "[SKIP-DRYRUN] $safe_cmd"; to_log "[SKIP-DRYRUN] $safe_cmd"; return 0; } out="$TMPDIR/cmd.$$.out" set +e; sh -c "$cmd" >"$out" 2>&1; rc=$?; set -e end="$(date +%s)"; dur=$((end - start)) if [ -s "$out" ]; then if [ "$VERBOSE" = "true" ]; then while IFS= read -r line; do say "[OUT] $(printf %s "$line" | redact_secrets)" done <"$out" fi while IFS= read -r line; do to_log "[OUT] $(printf %s "$line" | redact_secrets)" done <"$out" fi rm -f "$out" 2>/dev/null || true if [ $rc -ne 0 ]; then say "[FAIL][$dur s] $safe_cmd (rc=$rc)"; to_log "[FAIL][$dur s] $safe_cmd (rc=$rc)" [ -f "$LOG_FILE" ] && { say "---- LOG TAIL ($LOG_TAIL_LINES) ----"; tail -n "$LOG_TAIL_LINES" "$LOG_FILE" || true; } if [ "$KEEP_TMP_ON_ERROR" = "true" ]; then _cleanup_needed="no"; say "Keeping temp dir: $TMPDIR"; fi exit $rc else say "[OK][$dur s] $safe_cmd"; to_log "[OK][$dur s] $safe_cmd" fi } apt_install() { MISSING=""; for p in "$@"; do dpkg -s "$p" >/dev/null 2>&1 || MISSING="$MISSING $p"; done [ -z "$MISSING" ] && return 0 export DEBIAN_FRONTEND=noninteractive run "apt-get update -y" run "apt-get install -y $MISSING" } require_env() { var="$1"; eval "val=\${$var:-}"; [ -n "${val:-}" ] || die "missing required env: $var"; } ensure_root_or_sudo() { if is_root; then return; fi if need_cmd sudo; then section "Escalating privileges with sudo"; exec sudo -E sh -c "$(cat)"; fi section "Installing sudo"; apt_install sudo || true if need_cmd sudo; then exec sudo -E sh -c "$(cat)"; else die "need root privileges or sudo to continue."; fi } init_logging() { mkdir -p "$LOG_DIR" || true : >"$LOG_FILE" || die "cannot write log file: $LOG_FILE" chmod 0640 "$LOG_FILE" || true say "Logging to: $LOG_FILE"; to_log "Log start" } enable_debug() { if [ "${DEBUG:-false}" = "true" ]; then set -x; say "DEBUG tracing enabled (-x)"; to_log "DEBUG tracing enabled (-x)"; fi; } ############################################################################### # START # ############################################################################### ensure_root_or_sudo init_logging enable_debug section "Prechecks & dependencies (Ubuntu)" apt_install ca-certificates curl jq tar update-ca-certificates >/dev/null 2>&1 || true require_env CLUSTER_ID require_env TOKEN say "CLUSTER_ID=${CLUSTER_ID}" toklen="$(printf %s "$TOKEN" | wc -c | tr -d ' ')"; say "TOKEN provided (length: $toklen)" ############################################################################### # FETCH CLUSTER DETAILS FROM API ############################################################################### section "Fetching cluster metadata from API" TLS_FLAG="$(curl_tls_flag)" CURL_API="curl --location --insecure -H '$(auth_header)'" run "$CURL_API '$API_BASE_URL/v1/kubernetes-clusters/$CLUSTER_ID' -o '$API_CLUSTER_JSON'" # Check if cluster is already provisioned PROVISIONED="$(jq -r ' .provisioned // .Provisioned // .kubernetescluster.provisioned // .kubernetescluster.Provisioned // false ' "$API_CLUSTER_JSON")" if [ "$PROVISIONED" = "true" ]; then die "Cluster $CLUSTER_ID is already provisioned. Cannot continue." fi # Accept top-level or nested under .kubernetescluster, camelCase or snake_case CLUSTER_SOURCE_IP="$( jq -r ' .source_ip // .sourceIp // .kubernetescluster.source_ip // .kubernetescluster.sourceIp // empty ' "$API_CLUSTER_JSON" )" API_GW_PORT="$( jq -r ' .gw_server_service_port // .gwServerServicePort // .kubernetescluster.gw_server_service_port // .kubernetescluster.gwServerServicePort // 0 ' "$API_CLUSTER_JSON" )" [ -n "${CLUSTER_SOURCE_IP:-}" ] && say "API source_ip: $CLUSTER_SOURCE_IP" [ "${API_GW_PORT:-0}" != "0" ] && say "API gw_server_service_port: $API_GW_PORT" GW_PORT_EFFECTIVE="${LIQO_GW_SERVER_SERVICE_PORT:-$API_GW_PORT}" ############################################################################### # INSTALL K3s # ############################################################################### section "Installing k3s" export K3S_KUBECONFIG_MODE INSTALL_K3S_EXEC="--flannel-backend=${K3S_FLANNEL_BACKEND} --cluster-cidr=${K3S_CLUSTER_CIDR} ${K3S_INSTALL_DISABLES}" export INSTALL_K3S_EXEC [ -n "$K3S_INSTALL_CHANNEL" ] && export K3S_INSTALL_CHANNEL="$K3S_INSTALL_CHANNEL" run "curl -fsSL '$K3S_INSTALL_SCRIPT_URL' | sh -" export KUBECONFIG="$KUBECONFIG_PATH"; say "KUBECONFIG=$KUBECONFIG" section "Waiting for Kubernetes API (not node readiness)" run "until kubectl get --raw='/readyz' >/dev/null 2>&1; do sleep 2; done" run "kubectl version --client --output=yaml || kubectl version --client || true" ############################################################################### # INSTALL CALICO # ############################################################################### section "Installing Calico operator & CRDs (${CALICO_VERSION})" run "kubectl apply --server-side -f '$CALICO_OP_CRDS_URL'" run "kubectl apply -f '$CALICO_OPERATOR_URL'" run "kubectl -n tigera-operator rollout status deploy/tigera-operator --timeout='$CALICO_ROLLOUT_TIMEOUT' || true" section "Applying Calico default custom resources" run "kubectl apply -f '$CALICO_CUSTOM_RES_URL'" cat >"$CPATCH" <<'EOF' apiVersion: operator.tigera.io/v1 kind: Installation metadata: name: default spec: calicoNetwork: nodeAddressAutodetectionV4: skipInterface: liqo.* EOF section "Patching Calico Installation for k3s (skip liqo.*)" run "kubectl patch installation default -n tigera-operator --type merge --patch-file '$CPATCH'" run "kubectl -n calico-system rollout status ds/calico-node --timeout='$CALICO_ROLLOUT_TIMEOUT' || true" ############################################################################### # NOW wait for Nodes Ready # ############################################################################### section "Waiting for nodes to become Ready (after CNI)" run "kubectl wait node --all --for=condition=Ready --timeout='10m'" run "kubectl get nodes -o wide" run "kubectl -n calico-system get pods" ############################################################################### # INSTALL liqoctl # ############################################################################### section "Installing liqoctl (${LIQO_VERSION})" run "curl -fsSL '$LIQOCTL_TGZ_URL' | tar -xz -C '$TMPDIR'" [ -f "$TMPDIR/liqoctl" ] || die "liqoctl binary not found in archive" run "install -o root -g root -m 0755 '$TMPDIR/liqoctl' '$LIQOCTL_PATH'" ############################################################################### # INSTALL Liqo (control plane) # ############################################################################### section "Installing Liqo into local cluster (k3s)" # liqoctl install k3s --pod-cidr 192.168.0.0/16 --cluster-id run "'$LIQOCTL_PATH' install k3s --pod-cidr '$LIQO_POD_CIDR' --cluster-id '$CLUSTER_ID' --timeout '$LIQO_TIMEOUT'" # Best-effort readiness checks run "kubectl -n liqo-system rollout status deploy --timeout='$LIQO_TIMEOUT' || true" run "kubectl -n liqo-system get deploy,ds,pods || true" run "kubectl -n liqo get cm,svc || true" ############################################################################### # GENERATE PEERING KUBECONFIG (RPC) AND EXTRACT YAML FROM .config # ############################################################################### section "Generating peering kubeconfig via API (RPC endpoint)" run "$CURL_API -X POST -H 'Content-Type: application/json' --data '{}' \ '$API_BASE_URL/v1/kubernetes-clusters/$CLUSTER_ID:generatePeeringConfig' \ -o '$KPEERCFG_RAW'" # Extract YAML kubeconfig from JSON .config (with fallbacks) if head -n1 "$KPEERCFG_RAW" | grep -q '^{'; then run "jq -er '.config // .kubeconfig // .kubeConfig // .result.config // .data.config' '$KPEERCFG_RAW' > '$KPEERCFG'" else die "Expected JSON from generatePeeringConfig; got non-JSON. Raw saved at $KPEERCFG_RAW" fi run "chmod 600 '$KPEERCFG'" run "kubectl --kubeconfig '$KPEERCFG' config view >/dev/null" ############################################################################### # PEER # ############################################################################### section "Liqo peering" PEER_ARGS="--remote-kubeconfig '$KPEERCFG' --gw-server-service-type '$LIQO_GW_SERVICE_TYPE' --timeout '$LIQO_TIMEOUT'" if [ -n "${GW_PORT_EFFECTIVE:-}" ] && [ "$GW_PORT_EFFECTIVE" != "0" ]; then PEER_ARGS="$PEER_ARGS --gw-server-service-port '$GW_PORT_EFFECTIVE'" fi run "'$LIQOCTL_PATH' peer $PEER_ARGS" ############################################################################### # CREATE NAMESPACES & OFFLOAD THEM VIA LIQOCTL # ############################################################################### section "Ensuring namespaces exist: $NAMESPACES" for ns in $NAMESPACES; do run "kubectl get ns '$ns' >/dev/null 2>&1 || kubectl create namespace '$ns'" done section "Offloading namespaces with liqoctl" for ns in $NAMESPACES; do REMOTE_NS="${CLUSTER_ID}-${ns}" run "'$LIQOCTL_PATH' offload namespace '$ns' \ --namespace-mapping-strategy '$LIQO_OFFLOAD_STRATEGY' \ --remote-namespace-name '$REMOTE_NS' \ --selector 'liqo.io/remote-cluster-id=$NODERINGS_CLUSTER_NAME' \ --timeout '$LIQO_TIMEOUT'" done ############################################################################### # DONE # ############################################################################### section "Done" say "Install finished. Log saved at: $LOG_FILE" say "Temporary files cleaned." say "Kubeconfig uploaded successfully."