id	summary	reporter	owner	description	type	status	priority	milestone	component	version	resolution	keywords	cc
16944	svn -> git migration script	wiktorn	team	"Split to 4 scripts to make it easier to restart script on errors
* git_migrate-1-init.sh
{{{#!bash
#!/bin/sh

set -e

git init josm-tests
cd josm-tests
# create commit, to keep master branch clear
git commit -m 'init' --allow-empty

###
# define remotes
###

git svn init --trunk=. --prefix=svn_jmapviewer -R jmapviewer https://svn.openstreetmap.org/applications/viewer/jmapviewer
git svn init --trunk=. --prefix=svn_josm -R josm https://josm.openstreetmap.de/svn/trunk
# two separate remotes for commons as this is 2x ~10 minutes instead of 2+ houres to fetch
# git svn init --trunk=. --include-paths='commons/proper/jcs' --prefix=svn_commons_jcs/ -R svn_commons_jcs https://svn.apache.org/repos/asf/commons/proper/jcs/trunk/commons-jcs-core/
git svn init --trunk=. --prefix=svn_commons_jcs/ -R svn_commons_jcs https://svn.apache.org/repos/asf/commons/proper/jcs/trunk/commons-jcs-core/
# git svn init --trunk=. --prefix=svn_commons/ -R svn_commons https://svn.apache.org/repos/asf/
git svn init --trunk=trunk -b branches -t tags --prefix=svn_commons_logging/ -R svn_commons_logging https://svn.apache.org/repos/asf/commons/proper/logging/
git svn init --trunk=trunk -b branches -t tags --prefix=svn_commons_codec/ -R svn_commons_codec https://svn.apache.org/repos/asf/commons/proper/codec/
git svn init --trunk=trunk -b branches -t tags --prefix=svn_commons_bzip2/ -R svn_commons_bzip2 http://svn.apache.org/repos/asf/ant/core/

git svn init --trunk=. --prefix=svn_commons_bzip2_small/ -R svn_commons_bzip2_small http://svn.apache.org/repos/asf/ant/core/trunk/src/main/org/apache/tools/bzip2

git remote add commons_compress https://github.com/apache/commons-compress.git

echo Done

}}}


* git_migrate-2-fetch.sh
{{{#!bash
#!/bin/sh

set -e

cd josm-tests
###
# fetch from remotes
###
git svn fetch jmapviewer
git svn fetch josm
# git svn fetch --log-window-size=100000 svn_commons
git svn fetch --log-window-size=100000 svn_commons_jcs
git svn fetch --log-window-size=100000 svn_commons_logging
git svn fetch --log-window-size=100000 svn_commons_codec
# git svn fetch --log-window-size=100000 svn_commons_bzip2
git svn fetch --log-window-size=100000 svn_commons_bzip2_small
git fetch commons_compress

echo Done
}}}


* git_migrate-3-prepare.sh
{{{#!bash
#!/bin/sh

set -e

cd josm-tests
###
# Reorganize externals
###
# TODO - for all - squash all commits before introduction to JOSM to one
# remove files, on removal in JOSM

# JMapViewer
git branch -f svn_jmapviewer_rewrite remotes/svn_jmapviewertrunk
git filter-branch -f --prune-empty --tree-filter '
  if [ -d src/org/openstreetmap/gui/jmapviewer ] ; then
      git mv src src.new &&
      find . -depth 1 |
      grep -v ^./src.new | grep -v ^./.git$ |
      xargs -r git rm -fr &&
      mkdir -p src/org/openstreetmap/gui &&
      git add src/org/openstreetmap/gui &&
      git mv src.new/org/openstreetmap/gui/jmapviewer src/org/openstreetmap/gui &&
      git rm -rf --ignore-unmatch src.new
  else
      find . -depth 1 |
      grep -v ^./.git$ |
      xargs -r git rm -fr
  fi' \
  svn_jmapviewer_rewrite

# Apache Commons Compress
# Linearlize commons_compress history
git branch -f svn_commons_compress_rewrite remotes/commons_compress/master
# linearlize history
git filter-branch -f --parent-filter 'cut -f 2,3 -d "" ""' svn_commons_compress_rewrite
# Extract externals
git filter-branch -f --prune-empty --tree-filter '
  if [ -d src/main/java ] ; then
      git mv src src.new &&
      find . -depth 1 |
      grep -v ^./src.new | grep -v ^./.git$ |
      xargs -r git rm -fr &&
      git mv src.new/main/java src &&
      git rm -rf --ignore-unmatch src.new
  else
      find . -depth 1 |
      grep -v ^./.git$ |
      xargs -r git rm -fr
  fi' \
  svn_commons_compress_rewrite


# Apache Commons JCS
git branch -f svn_commons_jcstrunk_rewrite remotes/svn_commons_jcs/trunk
# git filter-branch -f --prune-empty --tree-filter '
#   if [ -d trunk/commons-jcs-core/src/main/java/ ] ; then
#       git rm -rf src ;
#       git mv trunk/commons-jcs-core/src/main/java src &&
#       find . -depth 1 |
#       grep -v ^./src$ | grep -v ^./.git$ |
#       xargs -r git rm -fr
#   else
#       find . -depth 1 |
#       grep -v ^./.git$ |
#       xargs -r git rm -fr
#   fi' \
#   svn_commons_jcstrunk_rewrite
git filter-branch -f --prune-empty --tree-filter '
  if [ -d src/main/java/ ] ; then
      git mv src src.new &&
      find . -depth 1 |
      grep -v ^./src.new | grep -v ^./.git$ |
      xargs -r git rm -fr &&
      git mv src.new/main/java src &&
      git rm -rf --ignore-unmatch src.new
  else
      find . -depth 1 |
      grep -v ^./.git$ |
      xargs -r git rm -fr
  fi' \
  svn_commons_jcstrunk_rewrite

# Apache Commons Logging
git branch -f svn_commons_loggingtrunk_rewrite remotes/svn_commons_logging/trunk
git filter-branch -f --prune-empty --tree-filter '
  if [ -d src/main/java ] ; then
      git mv src src.new &&
      find . -depth 1 |
      grep -v ^./src.new | grep -v ^./.git$ |
      xargs -r git rm -fr &&
      git mv src.new/main/java src &&
      git rm -rf --ignore-unmatch src.new
  else
      find . -depth 1 |
      grep -v ^./.git$ |
      xargs -r git rm -fr
  fi' \
  svn_commons_loggingtrunk_rewrite

# TODO: Apache Codecs
# svn_commons_codec
git branch -f svn_commons_codec_rewrite remotes/svn_commons_codec/trunk
git filter-branch -f --prune-empty --tree-filter '
  if [ -d src/main/java ] ; then
      git mv src src.new &&
      find . -depth 1 |
      grep -v ^./src.new | grep -v ^./.git$ |
      xargs -r git rm -fr &&
      git mv src.new/main/java src &&
      git rm -rf --ignore-unmatch src.new
  elif [ -d src/java ] ; then
      git mv src src.new &&
      find . -depth 1 |
      grep -v ^./src.new | grep -v ^./.git$ |
      xargs -r git rm -fr &&
      git mv src.new/java src &&
      git rm -rf --ignore-unmatch src.new
  else
      find . -depth 1 |
      grep -v ^./.git$ |
      xargs -r git rm -fr
  fi' \
  svn_commons_codec_rewrite

git branch -f svn_commons_bzip2_rewrite remotes/svn_commons_bzip2_small/trunk
git filter-branch -f --prune-empty --tree-filter '
  mkdir -p src/org/apache/tools/bzip2
  git mv *java src/org/apache/tools/bzip2
  ' \
  svn_commons_bzip2_rewrite

}}}


* git_migrate-4-adapt-history.sh
{{{#!bash
#!/bin/sh

set -e

cd josm-tests

for i in svn_jmapviewer svn_commons_jcstrunk svn_commons_loggingtrunk svn_commons_codec svn_commons_compress svn_commons_bzip2 ; do
  git branch -f ""${i}_clean"" ""${i}_rewrite""
  # git rebase $(git log --reverse --oneline --no-abbrev-commit ""${i}_clean"" | head -n 1 | cut -d ' ' -f 1) ""${i}_clean"" || exit 1
done

git branch -f svn_commons_compress_compressors_clean svn_commons_compress_rewrite
git branch -f svn_commons_compress_utils_clean svn_commons_compress_rewrite


function josm_ver_to_timestamp {
  git log --grep ""git-svn-id: https://josm.openstreetmap.de/svn/trunk@${1}"" --pretty=""format:%at"" svn_josmtrunk
}

function first_before {
  BRANCH=""${1}""
  TIMESTAMP=""${2}""
  COMMIT_NO=$(( $( git log --pretty=""format:%at %H"" ""${BRANCH}"" | sort -n |  awk -F ' ' ""\$1 > ${TIMESTAMP} {print NR} {} "" | head -n 1) - 1 ))
  git log --pretty=""format:%at %H"" ${BRANCH} | sort -n | awk -F ' ' ""NR == ${COMMIT_NO} {print \$2}""
}

function first_after {
  BRANCH=""${1}""
  TIMESTAMP=""${2}""
  #COMMIT_NO=$( git log --pretty=""format:%at %H"" ""${BRANCH}"" | sort -n |  awk -F ' ' ""\$1 < ${TIMESTAMP} {print \$2} {} "" | head -n 1)
  git log --pretty=""format:%at %H"" ""${BRANCH}"" | sort -n |  awk -F ' ' ""\$1 < ${TIMESTAMP} {print \$2} {} "" | tail -n 1
  #git log --pretty=""format:%at %H"" ${BRANCH} | sort -n | awk -F ' ' ""NR == ${COMMIT_NO} {print \$2}""
}

function extract_timestamp {
    echo $1 |  sed -E -e ""s%[^0-9]*([0-9]*)[^0-9]*.*%\1%""
}

function truncate_from_to {
    BRANCH=""$1""
    START_TS=""$2""
    STOP_TS=""$3""
    echo ""Working on ${BRANCH}""
    if [ -z ""${BRANCH}"" ] ; then
         echo No branch provided
         exit 1;
    fi
    if [ -z ""${START_TS}"" ] ; then
         echo no STATRT_TS provided
         exit 1;
    fi

    COMMIT_ID=$(first_before ""${BRANCH}"" ""${START_TS}"")

    if [ -z ""${COMMIT_ID}"" ] ; then
        echo Unable to find commit on ""${BRANCH}"" before ""${START_TS}""
        exit 1
    fi
    export BRANCH
    export START_TS
    export COMMIT_ID
    git filter-branch -f --commit-filter '
      if [ ""${GIT_COMMIT}"" == ""${COMMIT_ID}"" ] ; then
        GIT_AUTHOR_DATE=""${START_TS}"" git commit-tree -m ""Initial import of ${BRANCH}"" ""$@""
      elif [ $(echo ""${GIT_AUTHOR_DATE}"" | sed -E -e ""s%[^0-9]*([0-9]*)[^0-9]*.*%\1%"" )  -lt ""${START_TS}"" ] ; then
        skip_commit ""$@""
      else
        git commit-tree ""$@""
      fi
    ' ${BRANCH}


    if [ -n ""${STOP_TS}"" ] ; then
        COMMIT_ID=$(first_after ""${BRANCH}"" ""${STOP_TS}"")
        export COMMIT_ID
        export STOP_TS
        git filter-branch -f --tree-filter '
          if [ ""${GIT_COMMIT}"" == ""${COMMIT_ID}"" ]; then
             find . -depth 1 | grep -v ^./git$ | xargs -r git rm -rf
          fi
          ' --commit-filter '
          if [ ""${GIT_COMMIT}"" == ""${COMMIT_ID}"" ] ; then
            GIT_AUTHOR_DATE=""${STOP_TS}"" git commit-tree -m ""Remove ${BRANCH}"" ""$@""
          elif [ $(echo ""${GIT_AUTHOR_DATE}"" | sed -E -e ""s%[^0-9]*([0-9]*)[^0-9]*.*%\1%"" ) -gt ""${STOP_TS}"" ] ; then
              skip_commit ""$@""
          else
              git commit-tree ""$@""
          fi
        ' ${BRANCH}
    fi
}

###
# Clean compress_compressors
###
git filter-branch -f --tree-filter '
  find src/org/apache/commons/compress -depth 1 | grep -v compressors | xargs -r git rm -rf
' svn_commons_compress_compressors_clean

git filter-branch -f --tree-filter '
  find src/org/apache/commons/compress -depth 1 | grep -v utils | xargs -r git rm -rf
' svn_commons_compress_utils_clean

truncate_from_to svn_commons_codec_clean $(josm_ver_to_timestamp 2796) $(josm_ver_to_timestamp 8149)
truncate_from_to svn_commons_bzip2_clean $(josm_ver_to_timestamp 2796) $(josm_ver_to_timestamp 7867)
truncate_from_to svn_commons_compress_compressors_clean $(josm_ver_to_timestamp 7867) $(josm_ver_to_timestamp 12499)
truncate_from_to svn_commons_compress_utils_clean $(josm_ver_to_timestamp 11569) $(josm_ver_to_timestamp 12499)
truncate_from_to svn_commons_compress_clean $(josm_ver_to_timestamp 12500)
truncate_from_to svn_commons_jcstrunk_clean $(josm_ver_to_timestamp 8168)
truncate_from_to svn_commons_loggingtrunk_clean $(josm_ver_to_timestamp 8168)

# get history of svn externals:
# svn propget -R svn:externals src/org/apache
# then e.g.
# svn log src/org/apache/commons
# svn propget -r 7867 -R svn:externals src/org/apache
#  since 2796 codec - http://svn.apache.org/repos/asf/commons/proper/codec/trunk/src/java/org/apache/commons/codec
#             tools/bzip2       - http://svn.apache.org/repos/asf/ant/core/trunk/src/main/org/apache/tools/bzip2
#  since 4460 - codec - http://svn.apache.org/repos/asf/commons/proper/codec/trunk/src/main/java/org/apache/commons/codec
#  since 7867 - tools/bzip2 -> compress ->  http://svn.apache.org/repos/asf/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors
# since 8149 - no codec
# since 8168 - jcs - http://svn.apache.org/repos/asf/commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs
#            - logging - http://svn.apache.org/repos/asf/commons/proper/logging/trunk/src/main/java/org/apache/commons/logging logging
# since 8698 - compress ->  https://github.com/apache/commons-compress/trunk/src/main/java/org/apache/commons/compress/compressors
# since 11569 - utils -> https://github.com/apache/commons-compress/trunk/src/main/java/org/apache/commons/compress/utils
# since 12500 - compress -> https://github.com/apache/commons-compress/trunk/src/main/java/org/apache/commons/compress compress
}}}

* git_migrate-5-merge.sh
{{{#!bash
#!/bin/sh

set -e

cd josm-tests

###
# Merge JOSM repository with externals, commits ordered by date
###

# each git log needs echo at the end because first/last commit will be lost
(
  git log --pretty=""format:%at %H %s"" remotes/svn_josmtrunk ; echo
  git log --pretty=""format:%at %H %s"" svn_jmapviewer_clean ; echo
  git log --pretty=""format:%at %H %s"" svn_commons_jcstrunk_clean ; echo
  git log --pretty=""format:%at %H %s"" svn_commons_loggingtrunk_clean ; echo
  git log --pretty=""format:%at %H %s"" svn_commons_codec_clean ; echo
  git log --first-parent --pretty=""format:%at %H %s"" svn_commons_compress_clean ; echo
  git log --first-parent --pretty=""format:%at %H %s"" svn_commons_compress_compressors_clean; echo
  git log --first-parent --pretty=""format:%at %H %s"" svn_commons_compress_utils_clean ; echo
  git log --pretty=""format:%at %H %s"" svn_commons_bzip2_clean; echo
) | sort -n | cut -d ' ' -f2 | sed -e 's/^/p /' > ../rebase_config

git config merge.renamelimit 5000
git checkout master
git rebase --keep-empty -i $(git log --reverse --oneline --no-abbrev-commit | head -n 1 | cut -d ' ' -f 1)

# helpful one-liner: git diff --name-only --diff-filter=U | xargs git checkout  ...

# remove empty commits
git rebase $(git log --reverse --oneline --no-abbrev-commit | head -n 1 | cut -d ' ' -f 1)

###
# Move jar files to Git LFS
###
git lfs migrate import --include=""*.jar"" --include-ref=refs/heads/master
git reflog expire --expire-unreachable=now --all
git gc --prune=now
}}}"	enhancement	closed	normal		unspecified		worksforme	hack-weekend-2018-10 svn git migration script