WikiProject Sweden/Infra/tile.openstreetmap.se

From OpenStreetMap Wiki
Jump to: navigation, search

linux

Ubuntu 13.10

mapnik

cat >> /etc/apt/sources.list

deb http://ppa.launchpad.net/mapnik/nightly-2.3/ubuntu saucy main
deb-src http://ppa.launchpad.net/mapnik/nightly-2.3/ubuntu saucy main

^C

add-apt-repository ppa:mapnik/nightly-2.3
apt-get update

apt-get inetall apt-get install software-properties-common
apt-get install libmapnik
apt-get install libmapnik-dev

apt-get install mapnik-input-plugin-postgis

apt-get install python-mapnik

python
import mapnik
^D


mod_tile

apt-get install apache2
apt-get install apache2-threaded-dev apache2-mpm-prefork apache2-utils
apt-get install libagg-dev
apt-get install libfreetype6
apt-get install libfreetype6-dev


apt-get install   libcairo2 libcairo2-dev python-cairo python-cairo-dev 
apt-get install   libcairomm-1.0-1 libcairomm-1.0-dev 

apt-get install g++ autoconf
apt-get install make
apt-get install automake

cd ~/install
git clone https://github.com/openstreetmap/mod_tile.git
cd mod_tile
./autogen
./configure
./make
./make install
./make install-mod_tile
bash -c "echo 'LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so' > /etc/apache2/mods-available/tile.load"
a2enmod tile
service apache2 restart
nano /etc/apache2/sites-available/000-default.conf
# You can manually configure each tile set with AddTileConfig or AddTileMimeConfig.
# The first argument is the URL path relative to this virtual host
# under which a tile set is served. The second argument specifies the
# name of the tile set. This is used in the communication with renderd
# and is the directory under which (meta)tiles are stored on disk.
#
# By default (AddTileConfig) mod_tile assumes you are serving png files, however,
# mod_tile can also serve arbitrary other tile types such as javascript vector tiles,
# assuming the backend render daemon can handle the file type.
# To this purpose AddTileMimeConfig takes a 3rd agument, the file extension and it
# will guess the correct mimetype from it. If the mime type is not set correctly automatically,
# you need to use the configuration file route, where you can specify the mimetype and file extension
# independently.
# 
#    AddTileConfig /folder/ TileSetName
#    AddTileMimeConfig /folder2/ TileSetName2 js

# Alternatively (or in addition) you can load all the tile sets defined in the configuration file into this virtual host
    LoadTileConfigFile /etc/renderd.conf

# Timeout before giving up for a tile to be rendered
    ModTileRequestTimeout 2

# Timeout before giving up for a tile to be rendered that is otherwise missing
    ModTileMissingRequestTimeout 30

# If tile is out of date, don't re-render it if past this load threshold (users gets old tile)
    ModTileMaxLoadOld 2

# If tile is missing, don't render it if past this load threshold (user gets 404 error)
    ModTileMaxLoadMissing 500

# Socket where we connect to the rendering daemon
    ModTileRenderdSocketName /var/run/renderd.sock

##
## Options controlling the cache proxy expiry headers. All values are in seconds.
##
## Caching is both important to reduce the load and bandwidth of the server, as
## well as reduce the load time for the user. The site loads fastest if tiles can be
## taken from the users browser cache and no round trip through the internet is needed.
## With minutely or hourly updates, however there is a trade-off between cacheability
## and freshness. As one can't predict the future, these are only heuristics, that
## need tuning.

## If there is a known update schedule such as only using weekly planet dumps to update the db,
## this can also be taken into account through the constant PLANET_INTERVAL in render_config.h
## but requires a recompile of mod_tile

## The values in this sample configuration are not the same as the defaults
## that apply if the config settings are left out. The defaults are more conservative
## and disable most of the heuristics.


##
## Caching is always a trade-off between being up to date and reducing server load or
## client side latency and bandwidth requirements. Under some conditions, like poor
## network conditions it might be more important to have good caching rather than the latest tiles.
## Therefor the following config options allow to set a special hostheader for which the caching
## behaviour is different to the normal heuristics
##
## The CacheExtended parameters overwrite all other caching parameters (including CacheDurationMax)
## for tiles being requested via the hostname CacheExtendedHostname
#ModTileCacheExtendedHostname cache.tile.openstreetmap.org
#ModTileCacheExtendedDuration 2592000

# Upper bound on the length a tile will be set cacheable, which takes
# precedence over other settings of cacheing
ModTileCacheDurationMax 604800

# Sets the time tiles can be cached for that are known to by outdated and have been
# sent to renderd to be rerendered. This should be set to a value corresponding
# roughly to how long it will take renderd to get through its queue. There is an additional
# fuzz factor on top of this to not have all tiles expire at the same time
ModTileCacheDurationDirty 900

# Specify the minimum time mod_tile will set the cache expiry to for fresh tiles. There
# is an additional fuzz factor of between 0 and 3 hours on top of this.
ModTileCacheDurationMinimum 10800

# Lower zoom levels are less likely to change noticeable, so these could be cached for longer
# without users noticing much.
# The heuristic offers three levels of zoom, Low, Medium and High, for which different minimum
# cacheing times can be specified.

#Specify the zoom level below  which Medium starts and the time in seconds for which they can be cached
ModTileCacheDurationMediumZoom 13 86400

#Specify the zoom level below which Low starts and the time in seconds for which they can be cached
ModTileCacheDurationLowZoom 9 518400

# A further heuristic to determine cacheing times is when was the last time a tile has changed.
# If it hasn't changed for a while, it is less likely to change in the immediate future, so the
# tiles can be cached for longer.
# For example, if the factor is 0.20 and the tile hasn't changed in the last 5 days, it can be cached
# for up to one day without having to re-validate.
ModTileCacheLastModifiedFactor 0.20

## Tile Throttling
## Tile scrappers can often download large numbers of tiles and overly staining tileserver resources
## mod_tile therefore offers the ability to automatically throttle requests from ip addresses that have
## requested a lot of tiles.
## The mechanism uses a token bucket approach to shape traffic. I.e. there is an initial pool of n tiles
## per ip that can be requested arbitrarily fast. After that this pool gets filled up at a constant rate
## The algorithm has to metrics. One based on overall tiles served to an ip address and a second one based on
## the number of requests to renderd / tirex to render a new tile. 

## Overall enable or disable tile throttling
ModTileEnableTileThrottling Off
## Parameters (poolsize in tiles and topup rate in tiles per second) for throttling tile serving. 
ModTileThrottlingTiles 10000 1 
## Parameters (poolsize in tiles and topup rate in tiles per second) for throttling render requests. 
ModTileThrottlingRenders 128 0.2


</VirtualHost>

^XY

apt-get install libc-bin
ldconfig

mkdir /var/lib/mod_tile
mount /dev/vdb1 /var/lib/mod_tile
chown -R www-data:www-data /var/lib/mod_tile

renderd

cat > /etc/renderd.conf
[renderd]
stats_file=/var/run/renderd.stats
socketname=/var/run/renderd.sock
num_threads=16
tile_dir=/var/lib/mod_tile ; DOES NOT WORK YET

[mapnik]
plugins_dir=/usr/lib/mapnik/input
font_dir=/usr/share/fonts/truetype/ttf-dejavu
font_dir_recurse=false

#[default]
#URI=/osm/
#XML=/etc/mapnik-osm-data/default/osm.xml

[hydda_full]
URI=/hydda/full/
XML=/etc/mapnik-osm-data/hydda/hydda_full.xml

[hydda_base]
URI=/hydda/base/
XML=/etc/mapnik-osm-data/hydda/hydda_base.xml

[hydda_roads_and_labels]
URI=/hydda/roads_and_labels/
XML=/etc/mapnik-osm-data/hydda/hydda_roads_and_labels.xml

[unpaved_roads]
URI=/unpaved_roads/
XML=/etc/mapnik-osm-data/unpaved_roads.xml

^C

cat > /etc/init.d/renderd
#! /bin/sh
### BEGIN INIT INFO
# Provides:          Renderd
# Required-Start:    
# Required-Stop:     
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Example initscript
# Description:       This file should be used to construct scripts to be
#                    placed in /etc/init.d.
### END INIT INFO

# Author: kalle <kalle@openstreetmap.se>
#
# Please remove the "Author" lines above and replace them
# with your own name if you copy and modify this script.

# Do NOT "set -e"

# PATH should only include /usr/* if it runs after the mountnfs.sh script
PATH=/sbin:/usr/sbin:/bin:/usr/bin
DESC="Mapnik render deamon for mod tile in Apache httpd"
USER=www-data
NAME=renderd
DAEMON=/usr/local/bin/renderd
DAEMON_ARGS="-c /etc/renderd.conf"
PIDFILE=/var/run/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME

# Exit if the package is not installed
[ -x "$DAEMON" ] || exit 0

# Read configuration variable file if it is present
[ -r /etc/default/$NAME ] && . /etc/default/$NAME

# Load the VERBOSE setting and other rcS variables
. /lib/init/vars.sh

# Define LSB log_* functions.
# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
# and status_of_proc is working.
. /lib/lsb/init-functions

#
# Function that starts the daemon/service
#
do_start()
{
	# Return
	#   0 if daemon has been started
	#   1 if daemon was already running
	#   2 if daemon could not be started
	start-stop-daemon --start --quiet --pidfile $PIDFILE --user $USER \
		--background --exec $DAEMON --test > /dev/null \
		|| return 1
	start-stop-daemon --start --quiet --pidfile $PIDFILE --user $USER \
		--background --exec $DAEMON -- \
		$DAEMON_ARGS \
		|| return 2
	# Add code here, if necessary, that waits for the process to be ready
	# to handle requests from services started subsequently which depend
	# on this one.  As a last resort, sleep for some time.
}

#
# Function that stops the daemon/service
#
do_stop()
{
	# Return
	#   0 if daemon has been stopped
	#   1 if daemon was already stopped
	#   2 if daemon could not be stopped
	#   other if a failure occurred
	start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME
	RETVAL="$?"
	[ "$RETVAL" = 2 ] && return 2
	# Wait for children to finish too if this is a daemon that forks
	# and if the daemon is only ever run from this initscript.
	# If the above conditions are not satisfied then add some other code
	# that waits for the process to drop all resources that could be
	# needed by services started subsequently.  A last resort is to
	# sleep for some time.
	start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON
	[ "$?" = 2 ] && return 2
	# Many daemons don't delete their pidfiles when they exit.
	rm -f $PIDFILE
	return "$RETVAL"
}

#
# Function that sends a SIGHUP to the daemon/service
#
do_reload() {
	#
	# If the daemon can reload its configuration without
	# restarting (for example, when it is sent a SIGHUP),
	# then implement that here.
	#
	start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME
	return 0
}

case "$1" in
  start)
	[ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
	do_start
	case "$?" in
		0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
		2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
	esac
	;;
  stop)
	[ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
	do_stop
	case "$?" in
		0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
		2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
	esac
	;;
  status)
	status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
	;;
  #reload|force-reload)
	#
	# If do_reload() is not implemented then leave this commented out
	# and leave 'force-reload' as an alias for 'restart'.
	#
	#log_daemon_msg "Reloading $DESC" "$NAME"
	#do_reload
	#log_end_msg $?
	#;;
  restart|force-reload)
	#
	# If the "reload" option is implemented then remove the
	# 'force-reload' alias
	#
	log_daemon_msg "Restarting $DESC" "$NAME"
	do_stop
	case "$?" in
	  0|1)
		do_start
		case "$?" in
			0) log_end_msg 0 ;;
			1) log_end_msg 1 ;; # Old process is still running
			*) log_end_msg 1 ;; # Failed to start
		esac
		;;
	  *)
		# Failed to stop
		log_end_msg 1
		;;
	esac
	;;
  *)
	#echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2
	echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
	exit 3
	;;
esac

:

^C

chmod 755 /etc/init.d/renderd


renderd -f
^C
service renderd start

coastlines

skit i detta, kör skript och data per stil. om någon vill dela i framtiden så kan man hantera det då. blir ca 500MB overhead mellan hydda och osm-carto.


apt-get install unzip

root@modtile:~# cat > update-coastlines.sh 
# detta skript körs från crontab som root varje dag klockan 11:00
# egentligen skall man invalidera all cache efter det, men det blir lite dyrt och slöar ner maskinen.

mkdir /scratch/modtile
rm -rf /scratch/modtile/coastlines
mkdir /scratch/modtile/coastlines
cd /scratch/modtile/coastlines

# ersätter shorelines 300
wget http://data.openstreetmapdata.com/simplified-land-polygons-complete-3857.zip
unzip simplified-land-polygons-complete-3857.zip
cp -r simplified-land-polygons-complete-3857/* /usr/share/mapnik-osm-data/world_boundaries


# ersätter processed_p
wget http://data.openstreetmapdata.com/land-polygons-split-3857.zip
unzip land-polygons-split-3857.zip
cp -r land-polygons-split-3857/* /usr/share/mapnik-osm-data/world_boundaries

^C
root@modtile:~# chmod 755 update_coastlines.sh 
root@modtile:~# ./update_coastlines.sh

carto tiles


apt-get install mapnik-utils

apt-get install gdal-bin
apt-get install unzip


apt-get install npm
apt-get install nodejs-legacy

git clone https://github.com/mapbox/carto.git
cd carto
npm install -g carto

cd ..
git clone https://github.com/mapbox/millstone.git
cd millstone
npm install -g millstone


git clone https://github.com/gravitystorm/openstreetmap-carto.git
bash get-shapefiles.sh

cat > deploy.sh
#!/bin/bash
sed s/\"gis\"/\"gis\",\"host\":\"postgis\",\"user\":\"gis\"/g project.mml > target.mml
carto target.mml > project.xml

^C