Deploy and/or upgrade splunk forwarder on Ubuntu

Case: customer want simply wat to update splunk forwarder on Ubuntu servers.

Solution: script to simplify update process.

Solution

Create shell script and allow it to run

touch /opt/splunk_uf_upgrade.sh
chmod +x /opt/splunk_uf_upgrade.sh

script

#!/bin/bash
set -euo pipefail

/opt/splunkforwarder/bin/splunk stop || true
/opt/splunkforwarder/bin/splunk disable boot-start || true
cp -a /opt/splunkforwarder /opt/splunkforwarder_backup_$(date +%F) || true
dpkg -r splunkforwarder || true
rm -rf /opt/splunkforwarder
rm -rf /etc/systemd/system/SplunkForwarder.service
systemctl daemon-reexec
systemctl daemon-reload

# Download the latest .deb package
curl -s https://www.splunk.com/en_us/download/universal-forwarder.html | \
grep -Eo 'https://download.splunk.com/products/universalforwarder/releases/[0-9]+\.[0-9]+\.[0-9]+/linux/splunkforwarder-[0-9]+\.[0-9]+\.[0-9]+-[a-z0-9]+-linux-amd64\.deb' | \
head -n 1 | \
xargs -r wget -O splunkforwarder-latest.deb


# Install the .deb package
dpkg -i splunkforwarder-latest.deb

# Seed admin user
tee /opt/splunkforwarder/etc/system/local/user-seed.conf > /dev/null <<EOF
[user_info]
USERNAME = admin
PASSWORD = local_admin_password_here
EOF

# Ensure splunk group exists
getent group splunk > /dev/null || groupadd --system -g 119 splunk

# Ensure splunk user exists
id splunk &>/dev/null || useradd --system -u 119 -g 119 -d /opt/splunkforwarder -s /usr/sbin/nologin -c "splunk@$( hostname -s )" splunk

# Set permissions and config
chown -R splunk:splunk /opt/splunkforwarder
echo 'SPLUNK_BINDIP=127.0.0.1' | tee -a /opt/splunkforwarder/etc/splunk-launch.conf > /dev/null

# Start and enable service
sudo -u splunk /opt/splunkforwarder/bin/splunk start --accept-license --answer-yes --no-prompt
# Wait until Splunkd started
timeout 60 bash -c 'until sudo -u splunk /opt/splunkforwarder/bin/splunk status 2>/dev/null | grep -q "splunkd is running (PID"; do sleep 2; done'
echo "Confirmed splunkd is running — waiting 10 more seconds to ensure it's stable..."
sleep 10
/opt/splunkforwarder/bin/splunk stop
/opt/splunkforwarder/bin/splunk enable boot-start -user splunk
systemctl daemon-reload
systemctl enable SplunkForwarder
systemctl start SplunkForwarder

# Wait until Splunkd started
timeout 60 bash -c 'until sudo -u splunk /opt/splunkforwarder/bin/splunk status 2>/dev/null | grep -q "splunkd is running (PID"; do sleep 2; done'
# Confirm it's been running steadily for at least 10 seconds
echo "Confirmed splunkd is running — waiting 10 more seconds to ensure it's stable..."
sleep 10

# Authenticate first (must match what's in user-seed.conf)
sudo -u splunk /opt/splunkforwarder/bin/splunk login -auth admin:local_admin_password_here

# Configure forward-server and deployment-poll
sudo -u splunk /opt/splunkforwarder/bin/splunk add forward-server splunkidx.your_domain.ca:9997
sudo -u splunk /opt/splunkforwarder/bin/splunk set deploy-poll splunkdeploy.your_domain.ca:8089

# Verify UF version
echo
echo "== UF version =="
sudo -u splunk /opt/splunkforwarder/bin/splunk version 2>&1 | grep -v '^Warning'

# Verify configured deployment server
echo
echo "== Deployment Server Configuration =="
sudo -u splunk /opt/splunkforwarder/bin/splunk show deploy-poll 2>&1 | grep -v '^Warning'

# Verify configured forward servers
echo
echo "== Forward Servers =="
sudo -u splunk /opt/splunkforwarder/bin/splunk list forward-server 2>&1 | grep -v '^Warning'

echo
echo "== Installation completed =="

# Cleanup installer
rm -f splunkforwarder-latest.deb

run script and enjoy easy update

/opt/splunk_uf_upgrade.sh

done