OpenStack GhettoSAN on Raspberry Pi

“GhettoSAN” originally started out back when I wore a VMware hat more consistently and was a project born to answer: “I have 400,000 USB keys from a trade show, now what”. At first we ran Nexenta as a storage platform and while that ran well, however as it scaled up, we ran into issues with the number of USB hubs available to be presented to the VM, also Powering them.

A few months pass and I figure we’ve pushed this experiment to what is reasonable… then this comes along: http://programmerthoughts.com/openstack/swift-on-pi/ and I know we’re not done.

From here we’ll break this into two sections: Pics and Performance Numbers, Installation and Setup.

Pics and Performance Numbers

For those looking closely, yes, the Raspberry Pi is being powered from the hub it’s reading from.

Times

• Rebuilding the rings – 2min 38 sec
• Starting – 58min 12.231 sec
• Swift Bench… that’s when the magic smoke started. Sorry.

Installation and Setup

Partitioning and formatting ‘all’ the disks:

for i in `fdisk -l | grep /dev/sd | awk '{print $2}' | sed 's/://g'`; do (echo o; echo n; echo p; echo 1; echo ; echo; echo w) | sudo fdisk $i; done

 

for i in `fdisk -l | grep /dev/sd | awk '{print $1}' | grep /dev`; do mkfs.xfs $i -f; done

Then started with this: http://programmerthoughts.com/openstack/swift-on-pi/

But it needed to be adjusted like this:

#!/bin/bash

 

# This annotated script sets up a limited deployment of OpenStack Swift
# onto a Raspberry Pi. It sets up a one-replica, one-server environment
# appropriate for external testing. It assumes there is a user called "pi"
# and that user has sudo access (this is the default on a Raspberry Pi).

 

export DEBIAN_FRONTEND=noninteractive

 

set -e

 

# install requirements
# I assume you've already done an `apt-get update && apt-get upgrade`

 

sudo apt-get update && sudo apt-get upgrade -y

 

sudo apt-get install -y python-software-properties curl gcc git memcached 
 python-coverage python-dev python-nose python-setuptools 
 python-simplejson python-xattr sqlite3 xfsprogs python-eventlet 
 python-greenlet python-pastedeploy python-netifaces python-pip 
 python-sphinx
sudo pip install mock tox dnspython

 

# update /etc/fstab
for i in `sudo fdisk -l | grep /dev/sd | awk '{print $1}' | grep /dev | cut -d/ -f 3`;
do 
grep $i /etc/fstab
if [ $? = 1 ]; then
sudo tee -a /etc/fstab >/dev/null <<EOF
/dev/$i /mnt/$i xfs loop,noatime,nodiratime,nobarrier,inode64,logbufs=8 0 0
EOF
fi;
done

 

export C=1
for i in `sudo fdisk -l | grep /dev/sd | awk '{print $1}' | grep /dev | cut -d/ -f 3`;
do 
 sudo mkdir -p /mnt/$i/1
 sudo chown -R pi:pi /mnt/$i/1
 sudo ln -fs /mnt/$1/1 /srv/$C;
 C=$(( $C + 1 ))
done
sudo chown -R pi:pi /etc/swift /srv/* /var/run/swift

 

# update /etc/rc.local
grep 'su - pi /home/pi/bin/startmain' /etc/rc.local
if [ $? = 1 ]; then
sudo tee -a /etc/rc.local >/dev/null <<EOF

 

mkdir -p /var/cache/swift
chown pi:pi /var/cache/swift*
mkdir -p /var/run/swift
chown pi:pi /var/run/swift
su - pi /home/pi/bin/startmain
EOF
fi

 

sudo tee /etc/rsyncd.conf >/dev/null <<EOF
uid = pi
gid = pi
log file = /var/log/rsyncd.log
pid file = /var/run/rsyncd.pid
address = 127.0.0.1
EOF

 

for i in `ls /srv`; 
do
account=$((6002 + ($i * 10)))
container=$((6001 + ($i * 10)))
object=$((6000 + ($i * 10)))
sudo echo "
[account$account]
max connections = 25
path = /srv/$i/node/
read only = false
lock file = /var/lock/account6012.lock

 

[container$container]
max connections = 25
path = /srv/$i/node/
read only = false
lock file = /var/lock/container6011.lock

 

[object$object]
max connections = 25
path = /srv/$i/node/
read only = false
lock file = /var/lock/object6010.lock" >> /etc/rsyncd.conf;
done

 

sudo tee /etc/rsyslog.d/10-swift.conf >/dev/null <<EOF
# Uncomment the following to have a log containing all logs together
local1,local2,local3,local4,local5.* /var/log/swift/all.log

 

# Uncomment the following to have hourly proxy logs for stats processing
$template HourlyProxyLog,"/var/log/swift/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%"
local1.*;local1.!notice ?HourlyProxyLog

 

local1.*;local1.!notice /var/log/swift/proxy.log
local1.notice /var/log/swift/proxy.error
local1.* ~

 

local2.*;local2.!notice /var/log/swift/storage1.log
local2.notice /var/log/swift/storage1.error
local2.* ~
EOF

 

sudo mkdir -p /var/log/swift/hourly
sudo chmod -R g+w /var/log/swift

 

 

set +e
cd && git clone git://github.com/openstack/python-swiftclient.git
set -e
cd ~/python-swiftclient; git pull origin master && sudo python ./setup.py develop

 

set +e
cd && git clone git://github.com/openstack/swift.git
set -e
cd ~/swift; git pull origin master && sudo python ./setup.py develop

 

cd && mkdir -p ~/bin

 

sudo mkdir -p /etc/swift
sudo chown pi:pi /etc/swift

 

cat >/etc/swift/proxy-server.conf <<EOF
[DEFAULT]
bind_port = 8080
user = pi
log_facility = LOG_LOCAL1
log_level = DEBUG
eventlet_debug = true

 

[pipeline:main]
pipeline = catch_errors healthcheck proxy-logging cache slo ratelimit tempurl formpost tempauth staticweb container-quotas account-quotas proxy-logging proxy-server

 

[app:proxy-server]
use = egg:swift#proxy
allow_account_management = true
account_autocreate = true

 

[filter:tempauth]
use = egg:swift#tempauth
user_admin_admin = admin .admin .reseller_admin
user_test_tester = testing .admin
user_test2_tester2 = testing2 .admin
user_test4_tester4 = testing4 .admin
user_test_tester3 = testing3
user_demo_demo = demo .admin

 

[filter:catch_errors]
use = egg:swift#catch_errors

 

[filter:healthcheck]
use = egg:swift#healthcheck

 

[filter:cache]
use = egg:swift#memcache

 

[filter:proxy-logging]
use = egg:swift#proxy_logging

 

[filter:ratelimit]
use = egg:swift#ratelimit

 

[filter:domain_remap]
use = egg:swift#domain_remap

 

[filter:cname_lookup]
# Note: this middleware requires python-dnspython
use = egg:swift#cname_lookup

 

[filter:staticweb]
use = egg:swift#staticweb

 

[filter:formpost]
use = egg:swift#formpost

 

[filter:list-endpoints]
use = egg:swift#list_endpoints

 

[filter:bulk]
use = egg:swift#bulk

 

[filter:container-quotas]
use = egg:swift#container_quotas

 

[filter:account-quotas]
use = egg:swift#account_quotas

 

[filter:slo]
use = egg:swift#slo

 

[filter:tempurl]
use = egg:swift#tempurl

 

[filter:formpost]
use = egg:swift#formpost
EOF

 

sudo mkdir -p /etc/swift/account-server
sudo chown pi:pi /etc/swift/account-server

 

for i in `ls /srv`; 
do
sudo cat >/etc/swift/account-server/$i.conf <<EOF
[DEFAULT]
devices = /srv/$i/node/
bind_port = $((6002 + ($i * 10)))
user = pi
log_facility = LOG_LOCAL2
recon_cache_path = /var/cache/swift
eventlet_debug = true
log_level = DEBUG
mount_check = false
disable_fallocate = true

 

[pipeline:main]
pipeline = recon account-server

 

[app:account-server]
use = egg:swift#account

 

[filter:recon]
use = egg:swift#recon

 

[account-replicator]
vm_test_mode = yes

 

[account-auditor]

 

[account-reaper]
EOF
echo "done $i";
done

 

sudo mkdir -p /etc/swift/container-server
sudo chown pi:pi /etc/swift/container-server

 

for i in `ls /srv`; 
do
cat >/etc/swift/container-server/$i.conf <<EOF
[DEFAULT]
devices = /srv/$i/node/
bind_port = $((6001 + ($i *10)))
user = pi
log_facility = LOG_LOCAL2
recon_cache_path = /var/cache/swift
eventlet_debug = true
log_level = DEBUG
mount_check = false
disable_fallocate = true

 

[pipeline:main]
pipeline = recon container-server

 

[app:container-server]
use = egg:swift#container

 

[filter:recon]
use = egg:swift#recon

 

[container-replicator]
vm_test_mode = yes

 

[container-updater]

 

[container-auditor]

 

[container-sync]
EOF
echo "done $i";
done

 

sudo mkdir -p /etc/swift/object-server/
sudo chown pi:pi /etc/swift/object-server/

 

for i in `ls /srv`; 
do
cat >/etc/swift/object-server/$i.conf <<EOF
[DEFAULT]
devices = /srv/$i/node/
bind_port = $((6000 + ($i *10)))
user = pi
log_facility = LOG_LOCAL2
recon_cache_path = /var/cache/swift
eventlet_debug = true
log_level = DEBUG
mount_check = false
disable_fallocate = true

 

[pipeline:main]
pipeline = recon object-server

 

[app:object-server]
use = egg:swift#object

 

[filter:recon]
use = egg:swift#recon

 

[object-replicator]
vm_test_mode = yes

 

[object-updater]

 

[object-auditor]
EOF
echo "done $i";
done

 

# when setting up the hash_path_suffix, it is important to make it unique
# and keep it a secret
SUFF=`python -c 'import uuid; print uuid.uuid4().hex'`
cat <<EOF >/etc/swift/swift.conf
[swift-hash]
swift_hash_path_suffix = $SUFF

 

[swift-constraints]
#max_file_size = 5368709122
# Note: Since the Raspberry Pi has such limited storage space,
# the maximum size of a single object has been set to 500MB.
max_file_size = 524288000
#max_meta_name_length = 128
#max_meta_value_length = 256
#max_meta_count = 90
#max_meta_overall_size = 4096
#max_object_name_length = 1024
#container_listing_limit = 10000
#account_listing_limit = 10000
#max_account_name_length = 256
#max_container_name_length = 256
EOF

 

cat <<EOF >/home/pi/bin/remakerings
#!/bin/bash

 

cd /etc/swift

 

rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz

 

swift-ring-builder object.builder create 8 1 0
swift-ring-builder account.builder create 8 1 0
swift-ring-builder container.builder create 8 1 0
EOF

 

for i in `ls /srv`; 
do
account=$((6002 + ($i * 10)))
container=$((6001 + ($i * 10)))
object=$((6000 + ($i * 10)))

 

echo "
swift-ring-builder object.builder add r1z1-127.0.0.1:$object/d1 1
swift-ring-builder container.builder add r1z1-127.0.0.1:$container/d1 1
swift-ring-builder account.builder add r1z1-127.0.0.1:$account/d1 1" >> /home/pi/bin/remakerings;
done

 

echo "
swift-ring-builder object.builder rebalance
swift-ring-builder container.builder rebalance
swift-ring-builder account.builder rebalance
" >> /home/pi/bin/remakerings

 

chmod +x /home/pi/bin/*

 

cat <<EOF

 

===========================================

 

Install completed.

 

You can now call `resetswift` and `startmain` to clean everything and start
the Swift server processes.

 

To test, try the following:
export PIIP=<IP address of your Raspberry Pi>
curl -i -H "X-Auth-User: test:tester" -H "X-Auth-Key: testing" \
 http://${PIIP}:8080/auth/v1.0/
EOF