CodeSnippets: curl code

Never been to CodeSnippets before?

Snippets is a public source code repository. Easily build up your personal collection of code snippets, categorize them with tags / keywords, and share them with the world (or not, you can keep them private!)

Older Snippets »

9 total

Batch download code snippets

to shell osx mac curl bash unix code web download internet batch ed getopts snippets trap by jvs on Nov 12, 2008

Batch download snippets from http://codesnippets.joyent.com and convert them to text files using man textutil (available on Mac OS X 10.4 or later).

Note: Old snippet versions will be automatically replaced by the downloaded snippets without a backup!

Author: jv
License: The MIT License, Copyright (c) 2008 jv

Usage:

# usage: bds [-p num] [-t tag] [-u user] tag
bds vim
bds -p 1280
bds -u jvs
bds -t plistbuddy
bds -t tar
bds -t ipfw -u jvs

#!/opt/local/bin/bash

# "batch download snippets" from http://codesnippets.joyent.com and
# convert them to text files using man textutil (available on Mac OS X 10.4 or later).
#
# Note: Old snippet versions will be automatically replaced by the downloaded snippets without a backup!
#      An alternative to man textutil is html2text, http://www.mbayer.de/html2text/ (which is available via MacPorts).
#
# Author: jv
# License: The MIT License, http://www.opensource.org/licenses/mit-license.php
# Copyright (c) 2008 jv
#
# cat /usr/local/bin/bds
#
# usage: bds [-p num] [-t tag] [-u user] tag


declare BaseURL='http://codesnippets.joyent.com'
declare download_dir="${HOME}/Desktop/Snippets"

# make sure there is no trailing slash
BaseURL="${BaseURL%/}"
download_dir="${download_dir%/}"

declare BasePostURL="${BaseURL}/posts/show"
declare BaseTagURL="${BaseURL}/tag"
declare BaseUserURL="${BaseURL}/user"

# make sure there is no trailing slash
BasePostURL="${BasePostURL%/}"
BaseTagURL="${BaseTagURL%/}"
BaseUserURL="${BaseUserURL%/}"


# man textutil
declare InputEncoding='utf-8'
declare OutputEncoding='utf-8'

export IFS=$' \t\n'


# function to download a single post specified by a post number: bds -p num
# cf. snippet, http://codesnippets.joyent.com/posts/show/1282

function snippet() {

   declare NL OPWD file outputfile postnum title url

   if [[ "${1//[[:digit:]]/}" != "" ]]; then echo "Argument error. No positive integer: ${1}"; return 1; fi

   postnum="${1}"
   url="${BasePostURL}/${postnum}"
   download_dir="${download_dir}/single-downloads"
   /bin/mkdir -p "${download_dir}"
   OPWD="${PWD}"
   cd "${download_dir}"
   /usr/bin/curl -L -O -s --max-time 25 "${url}" || exit 1    # download snippet web page
   file="${download_dir}/${url##*/}"
   trap '/bin/rm -f "${file}"; exit 0' 0 1 2 13 15

   # get title of downloaded web page
   #title="$(/usr/bin/sed -E -n -e '/<[tT][iI][tT][lL][eE]>/{s/^.*<[tT][iI][tT][lL][eE]>(.*)<\/[tT][iI][tT][lL][eE]>.*$/\1/p;q;}' "${file}" | \
   #         /usr/bin/sed -E -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title

   title="$(/usr/bin/egrep -m 1 -io '<title>.*</title>' "${file}" | /usr/bin/sed -E -e 's/^<title>[[:space:]]*|[[:space:]]*<\/title>$//g' \
             -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title


   title="${title//CodeSnippets:/}"
   title="${title//\//:}"
   title="${title// /_}"
   title="${title//[[:cntrl:]]/}"
   title="${title%"${title##*[!_]}"}"   # remove trailing underscores

   if [[ $title == '_CodeDrive_Snippets_courtesy_of_Peter_Coopers_handy_little_app' ]] || [[ -z "$title" ]]; then
      printf "\e[0K\e[31m%s\e[0m:  %s\n" "couldn't access" "${url}"
      /bin/rm "${file}"
      return 1
   fi

   outputfile="${download_dir}/${postnum}_${title}.txt"
   #outputfile="${download_dir}/${title}.txt"  # without post number prefix
   #outputfile="${outputfile//__/_}"  # uniq underscores

   printf "\n\e[0K\e[1;30m%s\e[0m:  %s\n\n" "saved as" "${outputfile}"

   /usr/bin/textutil -output "${outputfile}" -convert txt -inputencoding "${InputEncoding}" -encoding "${OutputEncoding}" "${file}"
   /bin/rm "${file}"

   # escape backslashes
   # man bash 2>/dev/null | less -p 'Each command in a pipeline'
   #outputfile="$(printf "%q" "${outputfile}")"  # cf. help printf
   outputfile="${outputfile//\\/\\\\}"

   NL=$'\\\n'

cat <<EOF | /bin/ed -s "${outputfile}"
H
,g/Snippets is a public source code repository/1,/Snippets is a public source code repository/d
,g/You need to create an account or log in to post comments to this site//You need to create an account or log in to post comments to this site/,\$d
,g|(See related posts)$|s|.See related posts.|${NL}${NL}|
,g|^to.* by.* on .*[[:digit:]]$|s|^to\(.*\) by\(.*\) on \(.*[[:digit:]]\)$|${NL}${NL}Author:\2${NL}Date: \3${NL}URL: ${url}${NL}Tags:\1${NL}|
,g|^Comments on this post$|s|\(Comments on this post\)|${NL}\1:|
,g| posts on .* at |s|\(.* posts on .* at .*\)|${NL}\1:|
w
EOF

# additional ed commands
# delete line numbers
# ,g|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}|s|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}\(.*\)$|\1|
# delete range of lines
# 4,11d


   cd "${OPWD}"
   return 0

}



#----------------------------------------- end of function snippet



declare pflag tflag uflag
declare cnt count dir_name file no_posts_check NL OPWD outputfile postnum tagsite title url urls website 

if [[ $# -eq 0 ]]; then 
   printf "%s\n%s\n" 'No arguments given!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2
   exit 1
fi


while getopts ":p:t:u:" option
do
  case $option in
    p) pflag="$OPTARG" ;;
    t) tflag="$OPTARG" ;;
    u) uflag="$OPTARG" ;;
    [?]) printf "%s\n%s\n" 'Argument error!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2; exit 1;;
    *) ;;
  esac
done

shift $(($OPTIND - 1))


if [[ $# -eq 1 ]]; then

   dir_name="${1}"
   tagsite="${BaseTagURL}/${1}"

elif [[ $# -gt 1 ]]; then

   printf "%s\n%s\n" 'Too many arguments!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2
   exit 1

elif [[ -n "${pflag}" ]]; then
   snippet "${pflag}"
   exit 0

elif [[ -n "${tflag}" ]] && [[ -n "${uflag}" ]]; then

   dir_name="${tflag}-${uflag}"
   tagsite="${BaseUserURL}/${uflag}/tag/${tflag}"

elif [[ -n "${tflag}" ]]; then

   dir_name="${tflag}"
   tagsite="${BaseTagURL}/${tflag}"

elif [[ -n "${uflag}" ]]; then

   dir_name="${uflag}"
   tagsite="${BaseUserURL}/${uflag}"

else

   printf "%s\n%s\n" 'Argument error!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2
   exit 1

fi


tagsite="${tagsite%/}"

#echo $dir_name
#echo $tagsite

count=1
cnt=0
curl_max_time=20
website=''
no_posts_check=''
NL=$'\\\n'
download_dir="${download_dir}/${dir_name//\//:}"
download_dir="${download_dir%/}"
/bin/mkdir -p "${download_dir}"
OPWD="${PWD}"
cd "${download_dir}"

# print download directory
printf "\n\e[0K\e[1;30m%s\e[0m:  %s\n\n" "download directory" "${download_dir}"


while [[ -z "${no_posts_check}" ]]; do

   # download website of the form: 
   # http://somewebsite.com/tag/bash/1,
   # http://somewebsite.com/user/name/1 or 
   # http://somewebsite.com/user/name/tag/bash/1

   website="$(/usr/bin/curl -L -s --max-time $curl_max_time "${tagsite}/${count}" )"

   if [[ $? -ne 0 ]]; then 
      printf "\e[0K\e[31m%s\e[0m:  %s\n" "curl_max_time ${curl_max_time}" "${tagsite}/${count}"
      exit 1
   fi

   #if [[ -n "$(printf "%s" "${website}" | /usr/bin/egrep -o 'Application error \(Apache\)')" ]]; then 
      #no_posts_check='Application error (Apache)'
      #printf "\e[0K\e[31m%s\e[0m:  %s\n" "no further posts" "${no_posts_check}"
   #fi

   if [[ -n "$(printf "%s" "${website}" | /usr/bin/egrep -o '>No posts<')" ]]; then 
      no_posts_check='>No posts<'
      #printf "\e[0K\e[31m%s\e[0m:  %s\n" "no further posts" "${no_posts_check}"
   fi

: <<-'COMMENT'

   # works for Bash 3.0 or later
   if [[ "${website}" =~ '>No posts<' ]]; then 
      no_posts_check="${BASH_REMATCH[0]}"
      #printf "\e[0K\e[31m%s\e[0m:  %s\n" "no further posts" "${no_posts_check}"
   fi

COMMENT


   if [[ -z "${no_posts_check}" ]]; then

      # extract relevant post URLs
      #urls=( $(printf "%s\n" "${website}" | /usr/bin/sed -E -n -e "s|^.* href=\"(/posts/show/[[:digit:]]+)\".*$|${BaseURL}\1|p;g") )
      urls=( $(printf "%s\n" "${website}" | /usr/bin/egrep -o 'href="/posts/show/[[:digit:]]+"' | /usr/bin/sed -E -n -e "s|href=\"(/posts/show/[[:digit:]]+)\"|${BaseURL}\1|p;g") )

      for ((i=0; i < "${#urls[@]}"; i++)); do

         url="${urls[${i}]}"

         postnum="${url##*/}"
         file="${download_dir}/${postnum}"
         trap '/bin/rm -f "${file}"; exit 0' 0 1 2 13 15

         /usr/bin/curl -L -O -s --max-time $curl_max_time "${url}"

         if [[ $? -ne 0 ]]; then 
            printf "\e[0K\e[31m%s\e[0m:  %s\n" "curl_max_time ${curl_max_time}" "${url}"
            continue
         fi
 

         # get title of downloaded web page
         #title="$(/usr/bin/sed -E -n -e '/<[tT][iI][tT][lL][eE]>/{s/^.*<[tT][iI][tT][lL][eE]>(.*)<\/[tT][iI][tT][lL][eE]>.*$/\1/p;q;}' "${file}" | \
         #    /usr/bin/sed -E -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title

         title="$(/usr/bin/egrep -m 1 -io '<title>.*</title>' "${file}" | /usr/bin/sed -E -e 's/^<title>[[:space:]]*|[[:space:]]*<\/title>$//g' \
                -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title


         title="${title//CodeSnippets:/}"
         title="${title//\//:}"
         title="${title// /_}"
         title="${title//[[:cntrl:]]/}"
         title="${title%"${title##*[!_]}"}"   # remove trailing underscores

         #printf "%s\n" "${title}"

         if [[ $title == '_CodeDrive_Snippets_courtesy_of_Peter_Coopers_handy_little_app' ]] || [[ -z "$title" ]]; then
            printf "\e[0K\e[31m%s\e[0m:  %s\n" "couldn't access" "${url}"
            /bin/rm "${file}"
            continue
         fi

         outputfile="${download_dir}/${postnum}_${title}.txt"
         #outputfile="${download_dir}/${title}.txt"  # without post number prefix
         #outputfile="${outputfile//__/_}"  # uniq underscores

         let cnt++
         printf "\e[0K\e[1;32m%-6s\e[0m  %s\n" "${cnt}" "${outputfile##*/}"

         /usr/bin/textutil -output "${outputfile}" -convert txt -inputencoding "${InputEncoding}" -encoding "${OutputEncoding}" "${file}"

         /bin/rm "${file}"


         # escape backslashes
         # man bash 2>/dev/null | less -p 'Each command in a pipeline'
         #outputfile="$(printf "%q" "${outputfile}")"  # cf. help printf
         outputfile="${outputfile//\\/\\\\}"

# edit $outputfile in-place with man ed
# first delete lines at the beginning & end,
# then remove the string 'See related posts' and add some newlines with $NL,
# then convert the line 'to...by...on' to line 'Author:...', line 'Date:...', line 'URL:...' and line 'Tags:...'
# and finally the last two ed commands insert two further newlines with $NL

cat <<EOF | /bin/ed -s "${outputfile}"
H
,g/Snippets is a public source code repository/1,/Snippets is a public source code repository/d
,g/You need to create an account or log in to post comments to this site//You need to create an account or log in to post comments to this site/,\$d
,g|(See related posts)$|s|.See related posts.|${NL}${NL}|
,g|^to.* by.* on .*[[:digit:]]$|s|^to\(.*\) by\(.*\) on \(.*[[:digit:]]\)$|${NL}${NL}Author:\2${NL}Date: \3${NL}URL: ${url}${NL}Tags:\1${NL}|
,g|^Comments on this post$|s|\(Comments on this post\)|${NL}\1:|
,g| posts on .* at |s|\(.* posts on .* at .*\)|${NL}\1:|
w
EOF

# additional ed commands
# delete line numbers
# ,g|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}|s|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}\(.*\)$|\1|
# delete range of lines
# 4,11d


      done  # for

      let count++

   fi

done   # while


   cd "${OPWD}"


exit 0

Record your dynamic WAN IP addresses

to shell osx mac curl launchd bash ip address dynamic record wan dsl router wanip by jvs on Oct 15, 2008

A launchd + shell script exercise to record your DSL router's dynamic WAN IP addresses. Requires some customization on your part. Use at your own risk.

1. create the launchd item in /Library/LaunchDaemons

/usr/bin/sudo /bin/bash -c '

yourname=$(/usr/bin/logname)
LaunchdPlistFile="/Library/LaunchDaemons/net.${yourname}.wanip.update.plist"
EX_MARK='!'

/bin/cat > "${LaunchdPlistFile}" <<-EOF
<?xml version="1.0" encoding="UTF-8"?>
<${EX_MARK}DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
   <dict>
      <key>Disabled</key>
      <true/>
      <key>GroupName</key>
      <string>${yourname}</string>
      <key>Label</key>
      <string>net.${yourname}.wanip.update</string>
      <key>ProgramArguments</key>
      <array>
         <string>/Users/${yourname}/Library/wanip.sh</string>
      </array>
      <key>RunAtLoad</key>
      <true/>
      <key>StartInterval</key>
      <integer>20</integer>
      <key>UserName</key>
      <string>${yourname}</string>
   </dict>
</plist>
EOF
'


#-------------------------


yourname=$(/usr/bin/logname)
LaunchdPlistFile="/Library/LaunchDaemons/net.${yourname}.wanip.update.plist"

open -e "${LaunchdPlistFile}"
ls -l "${LaunchdPlistFile}"

/usr/bin/groups
/usr/bin/sudo /usr/sbin/chown root:wheel "${LaunchdPlistFile}"
#/usr/bin/sudo /usr/sbin/chown root:admin "${LaunchdPlistFile}"
/usr/bin/sudo /bin/chmod 0644 "${LaunchdPlistFile}"

# after creating ~/Library/wanip.sh below
/usr/bin/sudo /bin/launchctl load -w "${LaunchdPlistFile}" 2>/dev/null
#/usr/bin/sudo /bin/launchctl unload -w "${LaunchdPlistFile}" 2>/dev/null

/usr/bin/sudo /bin/launchctl list
/usr/bin/sudo /usr/bin/fs_usage | /usr/bin/egrep -i wanip

2. create a shell script that will be run by the launchd item at the specified intervals in seconds (here: every 20 seconds)

Version 1:

#!/bin/bash

# Version 1 with two columns (date, IP address)
# cat ~/Library/wanip.sh   (/Users/yourname/Library/wanip.sh)
# /usr/sbin/chown $(/usr/bin/logname):$(/usr/bin/logname) ~/Library/wanip.sh
# /bin/chmod 0744 ~/Library/wanip.sh

declare last_line_closed last_line_offline last_line_unreachable newfile old_wanip time wanip

declare IF='en0'
declare wanip_record_file="/Users/YOURLOGNAME/Library/wanip_record.txt"

# try to find your router_wanip_site by surfing to the IP addresses returned by the following commands:
# route -n get default | egrep interface | awk '{print $NF}'
# ipconfig getoption en0 router
# ipconfig getoption en0 domain_name_server

declare router_wanip_site='http://xxxx.xx/xxx.htm'
#declare router_wanip_site='http://checkip.dyndns.org'   # alternative


/bin/sleep 3

/usr/sbin/ipconfig waitall

if [[ "$(/sbin/route -n get default | /usr/bin/egrep interface | /usr/bin/awk '{print $NF}')" == "${IF}" ]]; then

   /usr/bin/curl -I -L -s --max-time 10 "${router_wanip_site}" 1>/dev/null
 
   if [[ $? -ne 0 ]]; then 
      time="$(/bin/date +%Y-%m-%d-%H.%M.%S-%Z)"
      /usr/bin/logger -i "${time}     router_wanip_site is unreachable for wanip.sh"
      last_line_unreachable="$(/usr/bin/sed -E -n -e '$,$s/^.+ (unreachable).*$/\1/p' "${wanip_record_file}")"
      if [[ -n "$last_line_unreachable" ]]; then exit 0; fi
      echo "${time}          router_wanip_site is unreachable" >> "${wanip_record_file}"
      exit 0
   fi


   # match first IP address with egrep
   wanip="$(/usr/bin/curl -L -s --max-time 10 "${router_wanip_site}" | \
              /usr/bin/egrep -o -m 1 ' ([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}')"
   wanip="${wanip// /}"

   # alternative with sed for matching a line with a characteristic string plus IP address
   #wanip="$(/usr/bin/curl -L -s --max-time 10 "${router_wanip_site}" | \
              #/usr/bin/sed -E -n -e '/STRING: /{s/^.+ ([[:digit:]\.]+).*$/\1/p;q;}')"


   time="$(/bin/date +%Y-%m-%d-%H.%M.%S-%Z)"

   if [[ -n "${wanip}" ]]; then 
      old_wanip="$(/usr/bin/sed -E -n -e '$,$s/^.+ ([[:digit:]\.]+).*$/\1/p' "${wanip_record_file}")"
      if [[ "${wanip}" == "${old_wanip}" ]]; then exit 0; fi
      echo "${time}          ${wanip}" >> "${wanip_record_file}"
   else
      last_line_closed="$(/usr/bin/sed -E -n -e '$,$s/^.+ (closed).*$/\1/p' "${wanip_record_file}")"
      if [[ -n "${last_line_closed}" ]]; then exit 0; fi
      echo "${time}          connection closed" >> "${wanip_record_file}"
   fi

else

   last_line_offline="$(/usr/bin/sed -E -n -e '$,$s/^.+ (offline).*$/\1/p' "${wanip_record_file}")"
   if [[ -n "$last_line_offline" ]]; then exit 0; fi
   time="$(/bin/date +%Y-%m-%d-%H.%M.%S-%Z)"
   echo "${time}          offline" >> "${wanip_record_file}"

fi

if [[ $(/usr/bin/stat -f %z "${wanip_record_file}") -gt 31457280 ]]; then
   newfile="${wanip_record_file}-$(/bin/date +%Y-%m-%d-%H.%M.%S-%Z)"
   /bin/mv "${wanip_record_file}" "${newfile}"
fi

exit 0

Version 2:

#!/bin/bash

# Version 2 has an additional $name column
# cat ~/Library/wanip.sh   (/Users/yourname/Library/wanip.sh)
# /usr/sbin/chown $(/usr/bin/logname):$(/usr/bin/logname) ~/Library/wanip.sh
# /bin/chmod 0744 ~/Library/wanip.sh

declare format last_line_closed last_line_offline last_line_unreachable name newfile old_name old_wanip time wanip

declare IF='en0'
declare wanip_record_file="/Users/YOURLOGNAME/Library/wanip_record.txt"

# try to find your router_wanip_site by surfing to the IP addresses returned by the following commands:
# route -n get default | egrep interface | awk '{print $NF}'
# ipconfig getoption en0 router
# ipconfig getoption en0 domain_name_server

declare router_wanip_site='http://xxxx.xx/xxx.htm'
#declare router_wanip_site='http://checkip.dyndns.org'   # alternative

name="$(/usr/bin/who | /usr/bin/awk '/console/ {print $1}')"
name="${name//[[:cntrl:]]/,}"
if [[ -z "${name}" ]]; then name='[logout]'; fi

old_name="$(/usr/bin/sed -E -n -e '$,$s/^[^ ]+ +([^ ]+) +[^ ].*$/\1/p' "${wanip_record_file}")"

format='%-35s%-20s%-20s\n'   # for printf

#/bin/sleep 3

/usr/sbin/ipconfig waitall

if [[ "$(/sbin/route -n get default | /usr/bin/egrep interface | /usr/bin/awk '{print $NF}')" == "${IF}" ]]; then

   /usr/bin/curl -I -L -s --max-time 10 "${router_wanip_site}" 1>/dev/null
 
   if [[ $? -ne 0 ]]; then 
      time="$(/bin/date +%Y-%m-%d-%H.%M.%S-%Z)"
      /usr/bin/logger -i "${time}     router_wanip_site is unreachable for wanip.sh"
      last_line_unreachable="$(/usr/bin/sed -E -n -e '$,$s/^.+ (unreachable).*$/\1/p' "${wanip_record_file}")"
      if [[ -n "$last_line_unreachable" ]]; then 
         if [[ "${name}" != "${old_name}" ]]; then
            printf "${format}" "${time}" "${name}" "router_wanip_site is unreachable" >> "${wanip_record_file}"
         fi
         exit 0
      fi
      printf "${format}" "${time}" "${name}" "router_wanip_site is unreachable" >> "${wanip_record_file}"
      exit 0
   fi


   # match first IP address with egrep
   wanip="$(/usr/bin/curl -L -s --max-time 10 "${router_wanip_site}" | \
              /usr/bin/egrep -o -m 1 ' ([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}')"
   wanip="${wanip// /}"

   # alternative with sed for matching a line with a characteristic string plus IP address
   #wanip="$(/usr/bin/curl -L -s --max-time 10 "${router_wanip_site}" | \
              #/usr/bin/sed -E -n -e '/STRING: /{s/^.+ ([[:digit:]\.]+).*$/\1/p;q;}')"


   time="$(/bin/date +%Y-%m-%d-%H.%M.%S-%Z)"

   if [[ -n "${wanip}" ]]; then 
      old_wanip="$(/usr/bin/sed -E -n -e '$,$s/^.+ ([[:digit:]\.]+).*$/\1/p' "${wanip_record_file}")"
      if [[ "${wanip}" == "${old_wanip}" ]]; then
         if [[ "${name}" != "${old_name}" ]]; then
            printf "${format}" "${time}" "${name}" "${wanip}" >> "${wanip_record_file}"
         fi
         exit 0
      fi

      printf "${format}" "${time}" "${name}" "${wanip}" >> "${wanip_record_file}"
   else
      last_line_closed="$(/usr/bin/sed -E -n -e '$,$s/^.+ (closed).*$/\1/p' "${wanip_record_file}")"
      if [[ -n "${last_line_closed}" ]]; then 
         if [[ "${name}" != "${old_name}" ]]; then
            printf "${format}" "${time}" "${name}" "connection closed" >> "${wanip_record_file}"
         fi
         exit 0
      fi
      printf "${format}" "${time}" "${name}" "connection closed" >> "${wanip_record_file}"
   fi

else

   last_line_offline="$(/usr/bin/sed -E -n -e '$,$s/^.+ (offline).*$/\1/p' "${wanip_record_file}")"
   time="$(/bin/date +%Y-%m-%d-%H.%M.%S-%Z)"
   if [[ -n "$last_line_offline" ]]; then 
         if [[ "${name}" != "${old_name}" ]]; then
            printf "${format}" "${time}" "${name}" "offline" >> "${wanip_record_file}"
         fi
      exit 0
   fi
   printf "${format}" "${time}" "${name}" "offline" >> "${wanip_record_file}"

fi


if [[ $(/usr/bin/stat -f %z "${wanip_record_file}") -gt 31457280 ]]; then
   newfile="${wanip_record_file}-$(/bin/date +%Y-%m-%d-%H.%M.%S-%Z)"
   /bin/mv "${wanip_record_file}" "${newfile}"
fi

exit 0

Version 3:

/usr/bin/sudo /bin/bash -c '

yourname=$(/usr/bin/logname)
LaunchdPlistFile="/Library/LaunchDaemons/net.${yourname}.wanip.update.plist"
EX_MARK='!'

/bin/cat > "${LaunchdPlistFile}" <<-EOF
<?xml version="1.0" encoding="UTF-8"?>
<${EX_MARK}DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
        <key>Disabled</key>
        <true/>
        <key>GroupName</key>
        <string>${yourname}</string>
        <key>Label</key>
        <string>net.${yourname}.wanip.update</string>
        <key>ProgramArguments</key>
        <array>
            <string>/Users/${yourname}/Library/wanip.sh</string>
        </array>
        <key>RunAtLoad</key>
        <true/>
        <key>UserName</key>
        <string>${yourname}</string>
</dict>
</plist>
EOF
'


#----------------------------------------------


yourname=$(/usr/bin/logname)
LaunchdPlistFile="/Library/LaunchDaemons/net.${yourname}.wanip.update.plist"

open -e "${LaunchdPlistFile}"
ls -l "${LaunchdPlistFile}"

/usr/bin/groups
/usr/bin/sudo /usr/sbin/chown root:wheel "${LaunchdPlistFile}"
#/usr/bin/sudo /usr/sbin/chown root:admin "${LaunchdPlistFile}"
/usr/bin/sudo /bin/chmod 0644 "${LaunchdPlistFile}"

# after creating ~/Library/wanip.sh below
/usr/bin/sudo /bin/launchctl load -w "${LaunchdPlistFile}" 2>/dev/null
#/usr/bin/sudo /bin/launchctl unload -w "${LaunchdPlistFile}" 2>/dev/null

/usr/bin/sudo /bin/launchctl list
/usr/bin/sudo /usr/bin/fs_usage | /usr/bin/egrep -i wanip


#-----------------------------------------------------------------------------------------------------


#!/bin/bash

# Version 3 is started only once by the launchd item and records shutdown processes
# Inspired by: "Re: how to run scripts at shutdown - how does launchd shutdown a system?", 
#              http://lists.apple.com/archives/macos-x-server/2007/Oct/msg00021.html
# cat ~/Library/wanip.sh   (/Users/yourname/Library/wanip.sh)
# /usr/sbin/chown $(/usr/bin/logname):$(/usr/bin/logname) ~/Library/wanip.sh
# /bin/chmod 0744 ~/Library/wanip.sh

declare last_line_closed last_line_offline last_line_unreachable name 
declare newfile old_name old_wanip printf_format time time_format wanip

declare IF='en0'
declare wanip_record_file="/Users/YOURLOGNAME/Library/wanip_record.txt"

# try to find your router_wanip_site by surfing to the IP addresses returned by the following commands:
# route -n get default | egrep interface | awk '{print $NF}'
# ipconfig getoption en0 router
# ipconfig getoption en0 domain_name_server

declare router_wanip_site='http://xxxx.xx/xxx.htm'
#declare router_wanip_site='http://checkip.dyndns.org'   # alternative


WHILEVAR=1
TRAPSIGNAL=

function exit_function() {
   TRAPSIGNAL='yes'
   # $! holds the PID of last process that has been started in the background (cmd &)
   [[ $! -gt $$ ]] && kill -TERM $!
}

trap exit_function SIGHUP SIGINT SIGTERM


time_format='+%Y-%m-%d--%H.%M.%S--%Z'
printf_format='%-40s%-20s%-20s\n'


while [[ ${WHILEVAR} ]]; do

   name="$(/usr/bin/who | /usr/bin/awk '/console/ {print $1}')"
   name="${name//[[:cntrl:]]/,}"
   if [[ -z "${name}" ]]; then name='[logout]'; fi
   old_name="$(/usr/bin/sed -E -n -e '$,$s/^[^ ]+ +([^ ]+) +[^ ].*$/\1/p' "${wanip_record_file}")"
   /usr/sbin/ipconfig waitall

   /bin/sleep 20 &     # time interval
   wait $!

   if [[ ${TRAPSIGNAL} ]]; then
      time="$(/bin/date ${time_format})"
      printf "${printf_format}" "${time}" "${name}" "shutdown" >> "${wanip_record_file}"
      exit 0
   fi


if [[ "$(/sbin/route -n get default | /usr/bin/egrep interface | /usr/bin/awk '{print $NF}')" == "${IF}" ]]; then

   /usr/bin/curl -I -L -s --max-time 10 "${router_wanip_site}" 1>/dev/null
 
   if [[ $? -ne 0 ]]; then 
      time="$(/bin/date ${time_format})"
      /usr/bin/logger -i "${time}     router_wanip_site is unreachable for wanip.sh"
      last_line_unreachable="$(/usr/bin/sed -E -n -e '$,$s/^.+ (unreachable).*$/\1/p' "${wanip_record_file}")"
      if [[ -n "$last_line_unreachable" ]]; then 
         if [[ "${name}" != "${old_name}" ]]; then
            printf "${printf_format}" "${time}" "${name}" "router_wanip_site is unreachable" >> "${wanip_record_file}"
         fi
         continue
      fi
      printf "${printf_format}" "${time}" "${name}" "router_wanip_site is unreachable" >> "${wanip_record_file}"
      continue
   fi


   # match first IP address with egrep
   wanip="$(/usr/bin/curl -L -s --max-time 10 "${router_wanip_site}" | \
              /usr/bin/egrep -o -m 1 ' ([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}')"
   wanip="${wanip// /}"

   # alternative with sed for matching a line with a characteristic string plus IP address
   #wanip="$(/usr/bin/curl -L -s --max-time 10 "${router_wanip_site}" | \
              #/usr/bin/sed -E -n -e '/STRING: /{s/^.+ ([[:digit:]\.]+).*$/\1/p;q;}')"


   time="$(/bin/date ${time_format})"

   if [[ -n "${wanip}" ]]; then 
      old_wanip="$(/usr/bin/sed -E -n -e '$,$s/^.+ ([[:digit:]\.]+).*$/\1/p' "${wanip_record_file}")"
      if [[ "${wanip}" == "${old_wanip}" ]]; then
         if [[ "${name}" != "${old_name}" ]]; then
            printf "${printf_format}" "${time}" "${name}" "${wanip}" >> "${wanip_record_file}"
         fi
         continue
      fi
      printf "${printf_format}" "${time}" "${name}" "${wanip}" >> "${wanip_record_file}"
   else
      last_line_closed="$(/usr/bin/sed -E -n -e '$,$s/^.+ (closed).*$/\1/p' "${wanip_record_file}")"
      if [[ -n "${last_line_closed}" ]]; then 
         if [[ "${name}" != "${old_name}" ]]; then
            printf "${printf_format}" "${time}" "${name}" "connection closed" >> "${wanip_record_file}"
         fi
         continue
      fi
      printf "${printf_format}" "${time}" "${name}" "connection closed" >> "${wanip_record_file}"
   fi

else

   last_line_offline="$(/usr/bin/sed -E -n -e '$,$s/^.+ (offline).*$/\1/p' "${wanip_record_file}")"
   time="$(/bin/date ${time_format})"
   if [[ -n "$last_line_offline" ]]; then 
         if [[ "${name}" != "${old_name}" ]]; then
            printf "${printf_format}" "${time}" "${name}" "offline" >> "${wanip_record_file}"
         fi
      continue
   fi
   printf "${printf_format}" "${time}" "${name}" "offline" >> "${wanip_record_file}"

fi


if [[ $(/usr/bin/stat -f %z "${wanip_record_file}") -gt 31457280 ]]; then
   newfile="${wanip_record_file}-$(/bin/date ${time_format})"
   /bin/mv "${wanip_record_file}" "${newfile}"
fi

done

exit 0

File upload with curl & AppleScript

to shell osx mac curl bash unix applescript ftp upload by jvs on Oct 14, 2008

upload a file by using curl + AppleScript

man curl 2>/dev/null | less -p '-T/--upload-file'
tell application "Finder" to do shell script "curl -T ~/resume.doc ftp://username:password@ftp.myserver.com/resume.doc"

Log website response times with cURL in windows

to time http curl windows cmd response dos batch by sahmeepee on May 06, 2008

This code snippet should be saved as a batch file and run in Windows. It can be set up as a scheduled task to log response times at a fixed interval. It takes one argument, the URL, which should be enclosed in quotes or Windows will barf on URLs with = (equals) signs in. If you don't supply any arguments, you will be prompted. Binary versions of curl are available via google.

If you have ISA:

REM measure response times for a site:
@echo off
IF a%1 == a (
  SET /P varHost=Enter the address, e.g. http://google.com: 
) ELSE (
  SET varHost=%1
)
SET startTime=%date% %time%
curl.exe --proxy-ntlm --proxy yourISAproxy:8080 --proxy-user username:password -s %varHost% > fulloutput.txt
echo %startTime%,%date% %time%,%varHost% >> respTimeLog.txt

If you have no proxy:

REM measure response times for a site:
@echo off
IF a%1 == a (
  SET /P varHost=Enter the address, e.g. http://google.com: 
) ELSE (
  SET varHost=%1
)
SET startTime=%date% %time%
curl -s %varHost% > fulloutput.txt
echo %startTime%,%date% %time%,%varHost% >> respTimeLog.txt

using curl

to curl commandline headers by standingwave on Dec 08, 2007

using curl to set content-type and post

curl -i -X POST -H "Content-Type: application/xml" -d @example.xml http://localhost:3000/example.xml

using curl to set Accept header

curl -i -H "Accept: application/xml" http://localhost:3000/example.xml

Facebook Status Updater using cURL

to php curl facebook by pwlin on Sep 08, 2007 1 Comments

<?PHP
/*******************************
*	Facebook Status Updater
*	Christian Flickinger
*	http://nexdot.net/blog
*	April 20, 2007
*******************************/

$status = 'YOUR_STATUS';
$first_name = 'YOUR_FIRST_NAME';
$login_email = 'YOUR_LOGIN_EMAIL';
$login_pass = 'YOUR_PASSWORD';

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://login.facebook.com/login.php?m&amp;next=http%3A%2F%2Fm.facebook.com%2Fhome.php');
curl_setopt($ch, CURLOPT_POSTFIELDS,'email='.urlencode($login_email).'&pass='.urlencode($login_pass).'&login=Login');
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_COOKIEJAR, "my_cookies.txt");
curl_setopt($ch, CURLOPT_COOKIEFILE, "my_cookies.txt");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.3) Gecko/20070309 Firefox/2.0.0.3");
curl_exec($ch);

curl_setopt($ch, CURLOPT_POST, 0);
curl_setopt($ch, CURLOPT_URL, 'http://m.facebook.com/home.php');
$page = curl_exec($ch);

curl_setopt($ch, CURLOPT_POST, 1);
preg_match('/name="post_form_id" value="(.*)" \/>'.ucfirst($first_name).'/', $page, $form_id);
curl_setopt($ch, CURLOPT_POSTFIELDS,'post_form_id='.$form_id[1].'&status='.urlencode($status).'&update=Update');
curl_setopt($ch, CURLOPT_URL, 'http://m.facebook.com/home.php');
curl_exec($ch);
?>

cURL Response Times

to curl ping response by kenzie on Jul 31, 2007

Determine response times of a URL with cURL.

echo "`curl -s -o /dev/null -w '%{time_starttransfer}-%{time_pretransfer}' http://www.domain.com/`"|bc

curl -w '\nLookup time:\t%{time_namelookup}\nConnect time:\t%{time_connect}\nPreXfer time:\t%{time_pretransfer}\nStartXfer time:\t%{time_starttransfer}\n\nTotal time:\t%{time_total}\n' -o /dev/null -s http://www.domain.com/

Downloading large files with cURL

to shell osx mac curl bash by jv on Jun 15, 2007

man curl



# broken downloads will be resumed automatically
cd ~/Desktop; while ! curl -C - -O 'http://download.parallels.com/GA/Parallels%20Desktop%203186%20Mac%20en.dmg'; do sleep 10; done


# requires FTP to be enabled in your firewall configuration
cd ~/Desktop; while ! curl -L -C - -O 'http://mirror.ctan.org/systems/mac/mactex/MacTeX.dmg'; do sleep 10; done

Source: Download with cURL in a Loop Until Completed

Upload all .gif and .jpg files in a directory to textpattern

to shell textpattern curl by Sencer on May 27, 2005 2 Comments

This will upload all files in the current directory ending in jpg jpeg or gif through your textpattern-admin panel to your site.

Save this as a shell script, adjust the first 3 variables and excute. (Note: LOGIN and PASS should be ascii only, or else manually url-encoded)

URL='http://www.mysite.com/textpattern/'
LOGIN='simplename'
PASS='yourpassword'

COOKIE=$(curl -s -D - -d "p_userid=$LOGIN&p_password=$PASS" \
    $URL | head -n10 | sed -n 's/^Set\-Cookie\: //p')

if [ -z $COOKIE ] 
then 
  echo "Can't log in."
  exit 1
else 
  echo "Cookie: "$COOKIE
fi


for file in $(ls -1|egrep '(gif)|(jpe?g)$') ; 
do
 echo "Sending "$file
 curl -s -H "Cookie: $COOKIE" -F "thefile=@$file" \
   -F "event=image" -F "step=image_insert" $URL > /dev/null
done

<strike>This will not output anything.</strike> But after it returns to the prompt, log in to textpattern and take a look into the image section.

Update: Added rudimentary check wether login is successful and basic progress meter.

Older Snippets »

9 total

CodeSnippets > curl

Related Tags

Never been to CodeSnippets before?

On This Page:

Batch download code snippets

Record your dynamic WAN IP addresses

File upload with curl & AppleScript

Log website response times with cURL in windows

using curl

Facebook Status Updater using cURL

cURL Response Times

Downloading large files with cURL

Upload all .gif and .jpg files in a directory to textpattern

On This Page:

Joyent

CodeSnippets > curl

Related Tags

Never been to CodeSnippets before?

On This Page:

On This Page: