# BigSun Full-Year 2014 Public Global Corpus
  * fy2014lst   84fe6b33e5b6f247..7543e9c7152f90e
  % part pdf-fy2014   844481d8e1cdb93e..c8be025c2f3d6a0
  % part misc-fy2014   04a660e4388f760b..66bf385d94eca1a
  + signed 0xC37FE5CA1AB690BD bigsun key   4ebaaee42aec6ecd..583e056dc295de5
  + signed 0x65A847E7C2B9380C coder key   77bf3b61c6fab52c..4bde0d8d04c4b47
  + signed 0x50CD9F782EC48CB3 paris14 key   25927313e36fece1..dc83072358f1e42
  = and this dist text, now known as fy2014.txt   e8e906316efadee2..65f25ffdf7ee97e


 ---


 ###
 ### This distribution is dedicated to Cryptome.org,
 ###  and prepared in memory of Aaron Swartz.
 ###
 
 Be sure to read the "Verification:" section before downloading.

 As the bulk of this release is sourced via Cryptome,
  a Cryptome Donation is Implored from Thee:
    >>>  http://cryptome.org/donations.htm

 "This is a trap, witting and unwitting.
  Do not use it or use at own risk.
  Source and this host is out to pwon and phuck you in complicity
   with global Internet authorities.
  ...
  Signed Batshit Cryptome and Host, 9 July 2014, 12:16ET."
   - cypherpunks/2014-July/005020.html, and true for FY-2014, too.

 Last but not least,
  Cryptome donation provides you a current dist, as this selection is not,
   and increasingly out of date by the day...


 ---


 # Gist of Dist - freely substitute any of full sources:
  bigsunt6eqt4tifc.onion bigsunymgzz5mdo3.onion bigsunr6tzikxo25.onion
  bigsundaawafn36e.onion bigsunklsutevagb.onion bigsunz7cmjr3hbd.onion
  bigsunkptoygblan.onion bigsun36arflx75h.onion bigsunndmif7ac5w.onion

 # And then?
 This archive is distributed ahead of processing upon it,
  and the odd structure of the corpus and distribution method
   will be explained in due time... for now all fifty gigs for fun!

 # Verification:
 As per convention,
  you must verify with detached signatures against verbatim copy,
   you must verify with at least the default bigsun signer.

 # bigsun key:
-----BEGIN PGP PUBLIC KEY BLOCK-----

mQSuBFSyw9sRDADgs+y6p016Mia20hY3VYLaX+vsRE7te5H+4RIGduPsbJ1L41rE
OJtiEInUubmWd+FUhu+ZvHjIMks16G71qc5w2FB0gIIcjGKaPnG3iGFLPiYbCXc6
w1EZGyBvW6Gd9nsKP2MODuXvQDtpP61gWHCyRzd2PM646BYT0lT9u+YIrJ4rayJn
ppkktOTc7Us7tSLL/wZdD91O3CE7UN2QKY/HJq44Ynpyry6gnTwSvSpDo2P/ypBP
9fBQRsCDPJbxJEpIlTkj6f4fx3GsNzBIQtU4AwkGxUtj/wAH0ARaYlIKyEdd2ign
GVnlcQlrNIXcAZyk+QXEDFssq1Z6uag9WpbPyOwvGRkxsq5ykSuaRnZCjEF3AYdn
OxCRMO6i/lBOYoExA+hiybh80vnRoJhhU5PIMM30BKAFECcvdGpDfQsO63Avvplp
M1PT4YP91pOqJ9aMQOpAnjzxEOHqPoy1dOYKhgP6UB82KPPm6Nr8jrucyd3dKf5/
9xylfxiw8xaVl1cBAJXdDoxfYPh5Yt09FyrVlDocPX7W9CAN6o0jqfBkwpynDACO
M8MoSTMD3f9h3n/MJ39EQkOrTe5YxVgERGv1g7F2OBwodRJxRaZN7gut8WM8NMmd
lB170KbJOfn7unpyK7HeSR9cnRCPuWHAnX54IikYp5WiCW53QVnOxJCzAyQKnzvE
PRp+5aNVr1cH5EDwf/5UxHnJtW5VSiFEq+RDdISfpUuxFUNxLCtjbPTh5x1dcLBQ
BQnNXMyjXvk5CMQ4+O9GATT7gGcta0avpy5WraFxcbk6i3TUBcYUbVQBie27N81z
7abn70flNjMmDhZrZm8Hz5FOuDIfd8KfYlcsw1ze9h9BqZuUR3dzYMLG8uTeg1Uw
qcDkrFhsUvJ7ByuJnFUijsiI8JLbUH3TkMb59+r9w45EQcnx8FroTIRrwgu+ELHy
DKc5kYbrpIymfNVsqSqxinJCZdJ+LyCcjJAIN3DEA6/3yj+jFNSHaFNg4UiWbFey
YLQh7EqrQOkPLJiECQS+DOwG/MF5VOnTHdLKopXBBIkIaXn469QnLGH7mHm8Ui4M
AJFsUNU/IbjXLYNCgPCz20V/1SaWMfrfdXBg4e4IvkE0JYAWjbgIJ1+hLV5/Ibps
AQUEHRw1aPkhQt7Pw6xUc1rozXhvc3AklF4+ZJrtK3Jrs3RtyHIyDND878vEE3Rj
hcPOjWw9GMUc9XRS9urmnEcq86voxdUoJu86s1sgKIns6WqjUlWeOIJbRMcE7/5U
nGxrIEAF87f7VLSP4Fc6PjWzf8J+QngP8896ju8hRB5qOFs0mfTPOUNMggnwJGLt
XiRb4idjYP7cu9qeviFRKjL5sk/lsP92jlNRx36dv1FyNFMsvJCZgHx/LA0jJiRj
uALlYkA1G/tWs1J/Utx/4xIdHBVjRdwc1Wd5K+eamdeZrB1nvUJRgLCDzDHjDx7t
+rhiMcSFIDoKs9BQ8dvfUStkmcGUXzf+hzyYNdgTd1tqJpt1d3hP7YMhU/TzjE1w
BM49VmsaRBY4OwbadfNbCQXg9aOvuQ8ShJM1Pv0QVycWeFQ6DOaRbBCiWk9HOE+n
s7YAAAEnQmlnU3VuIEZ1bGwtWWVhciAyMDE0IFB1YmxpYyBHbG9iYWwgQ29ycHVz
IChiaWdzdW5uZG1pZjdhYzV3Lm9uaW9uIGJpZ3N1bno3Y21qcjNoYmQub25pb24g
Ymlnc3VucjZ0emlreG8yNS5vbmlvbiBiaWdzdW4zNmFyZmx4NzVoLm9uaW9uIGJp
Z3N1bmtsc3V0ZXZhZ2Iub25pb24gYmlnc3VueW1neno1bWRvMy5vbmlvbiBiaWdz
dW5rcHRveWdibGFuLm9uaW9uIGJpZ3N1bmRhYXdhZm4zNmUub25pb24gYmlnc3Vu
dDZlcXQ0dGlmYy5vbmlvbikgPGFoeDJjaGFtZXRoZWkyb2h4YWlCNnVOZzJwaGFo
ZnVKQG5vLmV4aXN0Poh/BBMRCAAnBQJUssPbAhsDBQkB1ASABQsJCAcDBRUKCQgL
BRYDAgEAAh4BAheAAAoJEMN/5coatpC9/+AA/RGSnphqNwMCAQVdwP9MHUjrTE6v
KfZY3mGEkkO2DL4aAPwKVXFKct8arVR5EEISACdONWeGkyYOwe7Oq44ubrjLWw==
=j46s
-----END PGP PUBLIC KEY BLOCK-----

 # coder key:
  echo Key fingerprint = 41C1 2B8C 307D 7E21 98AA  5781 65A8 47E7 C2B9 380C
  echo "You may also search cypherpunks list and other forums for this ID."
  wget https://peertech.org/files/0x65A847E7C2B9380C-pub.txt

 # paris14 key:
  echo Key fingerprint = AE83 B248 B45F 7E12 4E5E  E6F6 50CD 9F78 2EC4 8CB3
  echo "See the Tor Developer Paris 2014 key signing for this key as well."
  curl https://peertech.org/files/0x50CD9F782EC48CB3.txt

 # all keys internally:
  ~/torsunget.sh 4ebaaee42aec6ecd731aae3c4135847ed947cc0c7bf9240a4583e056dc295de5 && gpg --import :)
  ~/torsunget.sh 77bf3b61c6fab52c69057a6d9f7f073f0738d4db35bfddb214bde0d8d04c4b47 && gpg --import :)
  ~/torsunget.sh 25927313e36fece1d0ac4e1bb3cc22b9dafa8a530f900fe2fdc83072358f1e42 && gpg --import :)

 # Detached Signature Verify Example E.g.:
  curl -x http://127.0.0.1:8778/ -o fy2014.txt http://bigsunz7cmjr3hbd/index.html
  curl -x http://127.0.0.1:8778/ -o fy2014.txt.asc http://bigsunz7cmjr3hbd/index.html.asc
  gpg --trusted-key 0xC37FE5CA1AB690BD --verify fy2014.txt.asc || mv fy2014.txt fy2014.txt.BAD

 # Verify ALL Signers Example E.g.:
  curl -x http://127.0.0.1:8778/ -o "fy2014.txt.forsigned" http://bigsunz7cmjr3hbd/fy2014.txt
  for signer in $(echo 0xC37FE5CA1AB690BD 0x65A847E7C2B9380C 0x50CD9F782EC48CB3); do 
    cp "fy2014.txt.forsigned" "fy2014.txt.${signer}"
    curl -x http://127.0.0.1:8778/ "http://bigsunz7cmjr3hbd/fy2014.txt.${signer}.asc"
    gpg --trusted-key "${signer}" --verify "fy2014.txt.${signer}.asc" || mv "fy2014.txt.${signer}" "fy2014.txt.${signer}.BAD"
  done

 # List all in BigSun Full-Year 2014 Public Global Corpus:
  - http://bigsun36arflx75h.onion/shid/84f/e6b/84fe6b33e5b6f247..7543e9c7152f90e
  - clearweb https://bigsunr6tzikxo25.tor2web.org/shid/84f/e6b/84fe6b33e5b6f247..7543e9c7152f90e
  - clearweb last resort https://peertech.org/files/fy2014lst.txt
  - suggest PDF and Misc parts before filling set with any missing.

 # Hint - Robust Downloads:
  - See next part, "Pull via aria2c:", for best results.
  - DO NOT use Tor Browser to download very large files!
  - *Please* use a download tool or plugin that supports HTTP 1.1 resume :)

 # PDFs in the BigSun Full-Year 2014 Public Global Corpus:
  - tar.xz http://bigsunklsutevagb.onion/shid/844/481/844481d8e1cdb93e..c8be025c2f3d6a0
  - clearweb tar.xz https://bigsunndmif7ac5w.tor2web.org/shid/844/481/844481d8e1cdb93e..c8be025c2f3d6a0
  - ~/torsunget.sh 844481d8e1cdb93ed4bfeb209da2513d7135795baff9b275bc8be025c2f3d6a0 && tar xJ :)

 # Misc documents in the BigSun Full-Year 2014 Public Global Corpus:
  - tar.xz http://bigsunkptoygblan.onion/
  - clearweb tar.xz https://bigsunymgzz5mdo3.tor2web.org/
  - ~/torsunget.sh 04a660e4388f760b07b2b47d7fbde83a6644734d8d20df74466bf385d94eca1a && tar xJ :)

 # Also:
  - http://sunshineeevvocqr.onion/bigsun/
  - clearweb https://sunshineeevvocqr.tor2web.org/bigsun/
  - clearweb http://cryptome.org/2014/12/peck-roark-affidavit.pdf


 ---


 # Preparation:

  echo copy the following lines into terminal, as commands
  apt-get install tor privoxy aria2
  nano /etc/privoxy/config

#--- begin-cut /etc/privoxy/config - replace text with nano editor ---
####
### - Tor Privoxy configuration - ALWAYS DOUBLE CHECK SETTINGS!!!
###   See "# for Tor browser bundle" section below, if needed.
###
### !NOTE! toggle=0 disables all privacy rewrite protections
###        only use this for aria2c from onions, if at all.
#
toggle  0
confdir /etc/privoxy
logdir /var/log/privoxy
logfile logfile
hostname hostname.example.org
listen-address  127.0.0.1:8778
enable-remote-toggle  0
enable-remote-http-toggle  0
enable-edit-actions 0
enforce-blocks 0
forwarded-connect-retries  0
accept-intercepted-requests 0
allow-cgi-request-crunching 0
split-large-forms 0
keep-alive-timeout 9
socket-timeout 900
max-client-connections 1024
#
# ENABLED DEFAULT for Tor upstream
forward-socks5   /               127.0.0.1:9050 .
#
# [disabled] option for Tor browser bundle Tor re-use
#forward-socks5   /               127.0.0.1:9150 .
#
#or other as desired...
#--- end-cut /etc/privoxy/config --- 


 # Pull via aria2c:

  # convention: called from ~ ($HOME), extracts to ~/shid/*/*/*
  nano ~/torsunget.sh && chmod +x ~/torsunget.sh
#--- begin-cut ~/torsunget.sh
#!/bin/bash
reqfile="${1}"
sha256="${2}"
outfile="${3}"
falen=$(echo -n $reqfile|wc -c)
if (( $falen == 64 )); then
  # special syntax for only sha-256 arg, note 15 tail incl. newline.
  export sha=$reqfile
  dirpr=$(echo $sha | head -c 3)
  sdirpr=$(echo $sha | head -c 6 | tail -c 3)
  fpr=$(echo $sha | head -c 16)
  fprt=$(echo $sha | tail -c 16)
  predir="shid/${dirpr}/${sdirpr}"
  mkdir -p "${predir}"
  outfile="${predir}/${fpr}..${fprt}"
  reqfile="${outfile}"
  sha256="$sha"
fi
export usehttpproxy="127.0.0.1:8778"
export onions="bigsunndmif7ac5w.onion bigsunz7cmjr3hbd.onion bigsunr6tzikxo25.onion bigsun36arflx75h.onion bigsunklsutevagb.onion bigsunymgzz5mdo3.onion bigsunkptoygblan.onion bigsundaawafn36e.onion bigsunt6eqt4tifc.onion" 
export olist="http://{"
for chost in `echo $onions`; do
  export olist="${olist}${chost},"
done
export olist="${olist}}/${reqfile}"
aria2c \
  --all-proxy="${usehttpproxy}" \
  --checksum="sha-256=${sha256}" \
  --continue=true --always-resume=true \
  --retry-wait=30 --timeout=120 \
  --summary-interval=3 \
  --piece-length=1M --min-split-size=1M \
  --max-connection-per-server=2 --max-concurrent-downloads=8 \
  -o "${outfile}" \
  --parameterized-uri=true \
  "${olist}"
if (( $? != 0 )); then
  exit $?
fi
#--- end-cut ~/torsunget.sh

  # these commands can be put in script, skip bundles as desired.
  export fylst="shid/84f/e6b/84fe6b33e5b6f247..7543e9c7152f90e=84fe6b33e5b6f2478523432514fcb24b844105c4a38635ba97543e9c7152f90e"
  pdfs="shid/844/481/844481d8e1cdb93e..c8be025c2f3d6a0=844481d8e1cdb93ed4bfeb209da2513d7135795baff9b275bc8be025c2f3d6a0"
  mscs="shid/04a/660/04a660e4388f760b..66bf385d94eca1a=04a660e4388f760b07b2b47d7fbde83a6644734d8d20df74466bf385d94eca1a"
  export bundles="${pdfs} ${mscs}"
  export files="${fylst} ${bundles}"
  for ctgt in `echo $files`; do
    cfile=$(echo $ctgt|sed 's/=.*//')
    cssum=$(echo $cfile|sed 's/.*[/]//')
    cssumdone="${cssum}.extracted"
    csum=$(echo $ctgt|sed 's/.*=//')
    if [ -f "${cssumdone}" ]; then
      echo "Skipping extracted bundle ${cssum} ..." >&2
    else
      echo "Retrieving $cfile ..."
      ~/torsunget.sh "${cfile}" "${csum}" "${cssum}"
    fi
  done
  for ctgt in `echo $bundles`; do
    cssum=$(echo $ctgt|sed 's/.*[/]//'|sed 's/=.*//')
    cssumdone="${cssum}.extracted"
    if [ -f "${cssum}" ]; then
      echo "Extracting bundle ${cssum} ..." >&2
      time cat "${cssum}" | tar xJ
      if (( $? == 0 )); then
        mv "${cssum}" "${cssumdone}"
      else
        now=$(date +%s)
        mv "${cssum}" "${cssum}.${now}.failed-extract"
        echo "Extracting ${cssum} failed, try again or upgrade tar program?" >&2
      fi
    fi
  done 
  if [ -f "${cssum}" ]; then
    echo "Starting download of remainder of full corpus ..." >&2
    cat "${cssum}"|sed 's/ .*//'|xargs -L 1 $HOME/torsunget.sh
    echo "Done. If any downloads failed, you must re-run this script."
  fi


 # Perform verification:

  nano ~/verifyshort.sh && chmod +x ~/verifyshort.sh
#--- begin-cut ~/verifyshort.sh
#!/bin/bash
fn="${2}"
mdest="${1}"
nofn=$(basename "$fn")
sha=$(sha256sum -- "$fn" | sed 's/ .*//')
dirpr=$(echo $sha | head -c 3)
sdirpr=$(echo $sha | head -c 6 | tail -c 3)
fpr=$(echo $sha | head -c 16)
fprt=$(echo $sha | tail -c 16)
predir="${dirpr}/${sdirpr}"
outfile="${predir}/${fpr}..${fprt}"
npt="${fpr}..${fprt}"
if [[ "$npt" != "$nofn" ]]; then
  now=$(date +%s)
  mfile="${mdest}/${nofn}-mod-${now}-${fpr}..${fprt}"
  echo "Saving modified $nofn to $mfile ..."
  cp -a "$fn" "$mfile"
fi
#--- end-cut ~/verifyshort.sh

  mkdir ~/malformed && cd ~/
  find shid -type f -exec $HOME/verifyshort.sh $HOME/malformed {} \;
  globlist=fy2014lst.txt
  if [ -f shid/84f/e6b/84fe6b33e5b6f247..7543e9c7152f90e ]; then
    globlist=shid/84f/e6b/84fe6b33e5b6f247..7543e9c7152f90e
  fi
  cat "$globlist"|sed 's/.* //'|xargs -L 1 ls |grep -v '^shid'
  if (( $? != 0 )); then
    echo "Some files missing from global corpus..." >&2
  fi


 # Next? Please be patient...