Resolve "refactor integrate_pandoc.sh"

Refactor the whole process in subroutines to allow for thorough testing - create_html - replace_marker - additional tests with against reference builds Furthermore, adapt pipeline to run all tests at testing stage

Resolve "refactor integrate_pandoc.sh"
Refactor the whole process in subroutines to allow for thorough testing - create_html - replace_marker - additional tests with against reference builds Furthermore, adapt pipeline to run all tests at testing stage
4ba90b28 · Fabian Nuraddin Alexander Gabel · 594ffcfb · 4ba90b28 · 4ba90b28 · 4ba90b28
Commit 4ba90b28 authored 3 years ago by Fabian Nuraddin Alexander Gabel
--- a/.gitignore
+++ b/.gitignore
-build/*
-*/build
+*build/
+*deploy/
 local/*
 public/*
 */public/*
@@ -8,3 +8,7 @@ public/*
 !static/home/*.html
 !static/topic/*.html
 *.log
+!testing/test_pandoc/*.html
+!testing/test_create_html/*.html
+*.swp
+!*ref_*
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,14 +3,14 @@
 #

 stages:
-  - preprocessor-test
+  - testing
  - build
  - review
  - deploy

-# always test the preprocessor and pandoc build with unit tests
-testing:
-  stage: preprocessor-test
+# always test the preprocessor with unit tests
+test-preprocessor:
+  stage: testing
  allow_failure: false
  image: 
    name: alpine:latest
@@ -23,10 +23,116 @@ testing:
    paths:
      - testing/*.log
  only:
-    - bin
-    - testing
+    refs:
+      - branches
+      - merge_requests
+    changes:
+      - .gitlab-ci.yml
+      - bin
+      - testing

-# build static webpage
+# always test the pandoc integration and pandoc build with unit tests
+pandoc-test:
+  stage: testing
+  allow_failure: false
+  image: 
+    name: alpine:latest
+  script:
+    - apk add --no-cache --upgrade bash
+    - RESEARCH_ROOT=`pwd`; export RESEARCH_ROOT;
+    # run shell script with testss
+    - $RESEARCH_ROOT/bin/test_pandoc.sh
+  artifacts:
+    paths:
+      - testing/test_pandoc/*.log
+
+# test that markdown is translated correctly 
+test-md-to-html:
+  stage: testing
+  allow_failure: false
+  image: 
+    name: eltenedor/ubuntu-pandoc:latest
+  script:
+    - mkdir -p log
+    - rm -rf log/test_create_html.log
+    - RESEARCH_ROOT=`pwd`; export RESEARCH_ROOT;
+    - $RESEARCH_ROOT/bin/test_create_html.sh | tee log/test_create_html.log
+  artifacts:
+    paths:
+      - log/test_create_html.log
+  only:
+    refs:
+      - branches
+      - merge_requests
+    changes:
+      - .gitlab-ci.yml
+      - bin
+      - testing
+
+# test that html is included correctly 
+test-replace_marker:
+  stage: testing
+  allow_failure: false
+  image: 
+    name: eltenedor/ubuntu-pandoc:latest
+  script:
+    - mkdir -p log
+    - rm -rf log/test_replace_marker.log
+    - RESEARCH_ROOT=`pwd`; export RESEARCH_ROOT;
+    - $RESEARCH_ROOT/bin/test_replace_marker.sh | tee log/test_replace_marker.log
+  artifacts:
+    paths:
+      - log/test_replace_marker.log
+  only:
+    refs:
+      - branches
+      - merge_requests
+    changes:
+      - .gitlab-ci.yml
+      - bin
+      - testing
+
+# test building standalone static webpage
+test-build-standalone:
+  stage: testing
+  allow_failure: false
+  image: 
+    name: eltenedor/ubuntu-pandoc:latest
+  script:
+    - mkdir -p log
+    - rm -rf log/test_build_standalone.log
+    - RESEARCH_ROOT=`pwd`; export RESEARCH_ROOT;
+    - $RESEARCH_ROOT/bin/test_build_standalone.sh | tee log/test_build_standalone.log
+  artifacts:
+    paths:
+      - log/test_build_standalone.log
+  only:
+    refs:
+      - branches
+      - merge_requests
+    changes:
+      - .gitlab-ci.yml
+      - bin
+      - testing
+
+# build standalone static webpage
+build-standalone-static:
+  stage: build
+  allow_failure: false
+  image: 
+    name: eltenedor/ubuntu-pandoc:latest
+  script:
+    - RESEARCH_ROOT=`pwd`; export RESEARCH_ROOT;
+    # build static webpage
+    - $RESEARCH_ROOT/bin/build_standalone.sh
+  artifacts:
+    paths:
+      - public 
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+    - if: $CI_COMMIT_BRANCH
+
+# build standalone static webpage
 build-static:
  stage: build
  allow_failure: false
@@ -43,8 +149,25 @@ build-static:
    - if: $CI_MERGE_REQUEST_ID
    - if: $CI_COMMIT_BRANCH

+# build tuhh-snippets
+build-tuhh:
+  stage: build
+  allow_failure: false
+  image: 
+    name: eltenedor/ubuntu-pandoc:latest
+  script:
+    - RESEARCH_ROOT=`pwd`; export RESEARCH_ROOT;
+    - $RESEARCH_ROOT/bin/preproc_topics.sh $RESEARCH_ROOT/build-tuhh $RESEARCH_ROOT/topics
+    - $RESEARCH_ROOT/bin/create_html.sh $RESEARCH_ROOT/build-tuhh $RESEARCH_ROOT/build-tuhh
+  artifacts:
+    paths:
+      - build-tuhh
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+    - if: $CI_COMMIT_BRANCH
+
 # deploy static webpage on issue-branch to web-server hosted asteroids
-asteroid:
+deploy-on-asteroid:
  stage: review
  image: 
    name: alpine:latest
@@ -62,7 +185,7 @@ asteroid:
    - chmod 700 ~/.ssh
  script:
    - mkdir .public-asteroid
-    - cp -r static/public/*.html .public-asteroid
+    - cp -r public/*.html .public-asteroid
    - cp -r img .public-asteroid
    - cp -r css .public-asteroid
    - mv .public-asteroid public-asteroid
@@ -78,8 +201,8 @@ asteroid:
    - when: always

 # deploy static webpage on dev-branch to tuhh
-tuhh-deploy:
-  stage: review
+deploy-on-tuhh:
+  stage: deploy
  image: 
    name: alpine:latest
  before_script:
@@ -93,10 +216,7 @@ tuhh-deploy:
    - echo "$TUHH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
    - chmod 700 ~/.ssh
  script:
-    - mkdir .public-tuhh
-    - cp -r static/public/*.html .public-tuhh
-    - cp -r img .public-tuhh
-    - cp -r css .public-tuhh
+    - cp -r build-tuhh .public-tuhh
    - mv .public-tuhh public-tuhh
    # deploy webpage on tuhh
    - echo 'scp -v public-tuhh' "$TUHH_USER@$TUHH_HOST:$TUHH_DEPLOY"
@@ -114,14 +234,12 @@ tuhh-deploy:
 pages:
  stage: deploy
  script:
-    - mkdir .public
-    - cp -r static/public/*.html .public
-    - cp -r img .public/.
-    - cp -r css .public/.
-    - mv .public public
+    - mkdir .public-pages
+    - cp -r public .public-pages
+    - mv .public-pages public-pages
  artifacts:
    paths:
-      - public
+      - public-pages
  rules:
    - if: $CI_COMMIT_BRANCH == 'master'

@@ -149,4 +267,4 @@ deploy_review:
 #    action: stop
 #  rules:
 #    - if: $CI_MERGE_REQUEST_ID
-#      when: manual
\ No newline at end of file
+#      when: manual
--- a/bin/adapt_html.sh
+++ b/bin/adapt_html.sh
@@ -36,3 +36,4 @@ do
    sed -i "s;/include/javascript/jquery.quicksilver.js;https://www.mat.tuhh.de/include/javascript/toggleme.js;" $f
    sed -i "s;/include/javascript/jquery.simpleFAQ.js;https://www.mat.tuhh.de/include/javascript/jquery.simpleFAQ.js;" $f
 done
+
--- a/bin/buildPandoc.sh
+++ b/bin/buildPandoc.sh
@@ -13,3 +13,4 @@ do
    -c "css/superfish-vertical.css" \
    -c "css/base_mode.css"
 done
+
--- a/bin/buildWebpage.sh
+++ b/bin/buildWebpage.sh
@@ -20,3 +20,4 @@ cp -r $RESEARCH_ROOT/css $DEPLOY_DIR/.public
 mv $DEPLOY_DIR/.public $DEPLOY_DIR/public

 echo "Done. Visit file://$DEPLOY_DIR/public/index.html ."
+
--- a/bin/build_standalone.sh
+++ b/bin/build_standalone.sh
+#!/bin/bash
+# building the a standalone static webpage
+RESEARCH_BIN=$RESEARCH_ROOT/bin
+RESEARCH_STATIC=$RESEARCH_ROOT/static
+
+# Check commandline arguments
+if [ $# -lt 1 ]
+then
+    RESEARCH_DEPLOY=`pwd`
+    RESEARCH_TOPICS=`pwd`"/topics"
+    RESEARCH_BUILD=`pwd`"/build"
+    echo -e "No deploy directory was specified."
+    # clean up directories
+    rm -rf $RESEARCH_BUILD
+    rm -rf $RESEARCH_DEPLOY/{.public,public}
+elif [ $# -lt 2 ] 
+then
+    RESEARCH_BUILD="$1"
+    RESEARCH_TOPICS=$RESEARCH_BUILD
+    RESEARCH_BUILD="$1/build"
+    echo -e "No extra topics directory was specified."
+    # clean up directories
+    rm -rf $RESEARCH_BUILD
+    rm -rf $RESEARCH_DEPLOY/{.public,public}
+elif [ $# -lt 3 ] 
+then
+    RESEARCH_DEPLOY="$1"
+    RESEARCH_TOPICS="$2"
+    RESEARCH_BUILD="$1/build"
+    echo -e "No extra build directory was specified."
+    # clean up directories
+    rm -rf $RESEARCH_BUILD
+    rm -rf $RESEARCH_DEPLOY/{.public,public}
+else
+    RESEARCH_DEPLOY="$1"
+    RESEARCH_TOPICS="$2"
+    RESEARCH_BUILD="$3"
+fi
+
+# prepare directories / clean-up
+mkdir -p $RESEARCH_DEPLOY
+mkdir -p $RESEARCH_BUILD
+cp -vf $RESEARCH_TOPICS/*.md $RESEARCH_BUILD
+
+echo -e "Deploying to $RESEARCH_DEPLOY ...;"
+echo -e "Building into directory $RESEARCH_BUILD ..."
+echo -e "Using .md-files from $RESEARCH_TOPICS ..."
+
+# preprocess .md-files 
+echo "Preproc Topics"
+$RESEARCH_BIN/preproc_topics.sh $RESEARCH_BUILD $RESEARCH_TOPICS
+
+# build html files
+echo "Creating HTML Files with pandoc"
+$RESEARCH_BIN/create_html.sh $RESEARCH_BUILD 
+
+# replace markers
+echo "Replacing markers"
+$RESEARCH_BIN/replace_marker.sh $RESEARCH_BUILD 
+
+#make public directory and copy files
+mkdir -p $RESEARCH_DEPLOY/.public
+for f in $RESEARCH_BUILD/*_final.html
+do
+    filename=$(basename -- "$f")
+    cp $f $RESEARCH_DEPLOY/.public/${filename%_final.html}.html
+done
+
+echo "Copying files to final destination ..."
+cp -r $RESEARCH_ROOT/{img,css}  $RESEARCH_DEPLOY/.public
+mv -v $RESEARCH_DEPLOY/.public  $RESEARCH_DEPLOY/public
+
+echo "Done. Visit file://$RESEARCH_DEPLOY/public/index.html ."
+
--- a/bin/build_static.sh
+++ b/bin/build_static.sh
@@ -8,9 +8,16 @@ BIN_DIR=$RESEARCH_ROOT/bin
 rm -rf  $BUILD_DIR
 rm -rf $DEPLOY_DIR/{.public,public}

-#build html 
+# preprocess .md-files 
 echo "Preproc Topics"
-$BIN_DIR/preproc_topics.sh $RESEARCH_ROOT/static
+$BIN_DIR/preproc_topics.sh $RESEARCH_ROOT/static $RESEARCH_ROOT/topics
+
+# get full TUHH staff pages ready for insertion
+mkdir -p $RESEARCH_ROOT/static/build/plain
+cp $RESEARCH_ROOT/static/home/*.html $RESEARCH_ROOT/static/build/plain
+cp $RESEARCH_ROOT/static/forschung/*.html $RESEARCH_ROOT/static/build/plain
+
+# build html files
 echo "Run Pandoc"
 $BIN_DIR/integrate_pandoc.sh
 echo "Pandoc finished."
@@ -27,3 +34,4 @@ cp -r $RESEARCH_ROOT/css $DEPLOY_DIR/.public
 mv $DEPLOY_DIR/.public $DEPLOY_DIR/public

 echo "Done. Visit file://$DEPLOY_DIR/public/index.html ."
+
--- a/bin/create-staff-yml.sh
+++ b/bin/create-staff-yml.sh
 #!/bin/bash
-    namelist=`grep -h -i -m 1 -r "###\s*Collaborators (MAT):" topics/topic-template.md | sed -e 's/^###\s*Collaborators (MAT):\s*//I' -e 's/\s*,\s*/\n/g'  | sort -u`
-    for name in $namelist
-    do
-        # pipeline to get full name of staff-member from mat-homepage
-        # -> wget the staff homepage of $name
-        # -> grep the line with the <h1>-tag, something like <h1>Fabian Gabel, M. Sc.</h1><div class='staffIntro'><p><img src='/home/fgabel/images/portrait.png' title='Foto von Fabian Gabel, M. Sc.' class='staffPicture'></p><div class='staffContact'>
-        # -> strip the string such that only the portion between <h1></h1> remains
-        # -> remove leading spaces
-        
-        echo "Found collaborator $name"
+namelist=`grep -h -i -m 1 -r "###\s*Collaborators (MAT):" topics/topic-template.md | sed -e 's/^###\s*Collaborators (MAT):\s*//I' -e 's/\s*,\s*/\n/g'  | sort -u`
+for name in $namelist
+do
+    # pipeline to get full name of staff-member from mat-homepage
+    # -> wget the staff homepage of $name
+    # -> grep the line with the <h1>-tag, something like <h1>Fabian Gabel, M. Sc.</h1><div class='staffIntro'><p><img src='/home/fgabel/images/portrait.png' title='Foto von Fabian Gabel, M. Sc.' class='staffPicture'></p><div class='staffContact'>
+    # -> strip the string such that only the portion between <h1></h1> remains
+    # -> remove leading spaces
+    
+    echo "Found collaborator $name"

-        # replace name in .md file
-        echo "- pandoc --standalone ./build/$name.md -o ./build/$name.html --mathjax" >> out.yml
+    # replace name in .md file
+    echo "- pandoc --standalone ./build/$name.md -o ./build/$name.html --mathjax" >> out.yml
+
+done

-    done
\ No newline at end of file
--- a/bin/create_html.sh
+++ b/bin/create_html.sh
+#!/bin/bash
+# processing command line arguments
+# Define variables
+MATHJAX_URL='https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js'
+
+# Check commandline arguments
+if [ $# -lt 1 ]
+then
+    RESEARCH_BUILD=`pwd`"/build"
+    RESEARCH_TOPICS=`pwd`
+    echo -e "No building directory was specified."
+elif [ $# -lt 2 ] 
+then
+    RESEARCH_BUILD="$1"
+    RESEARCH_TOPICS=$RESEARCH_BUILD
+    echo -e "No extra topics directory was specified."
+else
+    RESEARCH_BUILD="$1"
+    RESEARCH_TOPICS="$2"
+fi
+mkdir -p $RESEARCH_BUILD
+echo -e "Building into directory $RESEARCH_BUILD ..."
+echo -e "Using .md-files from $RESEARCH_TOPICS ..."
+
+# specify URL to avoid use of local library
+# see https://docs.mathjax.org/en/latest/web/start.html#ways-of-accessing-mathjax
+# and https://www.jsdelivr.com/package/npm/mathjax
+
+# create plain _research.html to be integrated into other files
+for f in $RESEARCH_TOPICS/*.md
+do
+    filename=$(basename -- "$f")
+    echo "Creating " $RESEARCH_BUILD/"${filename%.md}"_research.html " ..."
+    pandoc  $f -o  $RESEARCH_BUILD/"${filename%.md}"_research.html \
+    --mathjax=$MATHJAX_URL \
+    -c css/base.css \
+    -c css/extra.css \
+    -c "css/print.css" \
+    -c "css/superfish.css" \
+    -c "css/superfish-vertical.css" \
+    -c "css/base_mode.css"
+done
+
--- a/bin/extract_title.sh
+++ b/bin/extract_title.sh
@@ -3,4 +3,5 @@

 filename=$(basename -- "$1")

-echo "## `head -n 1 $1 | sed -e "s;#\s*;\[;g" -e "s;\(.*\);\1\](${filename%.md}.html);g"`"
\ No newline at end of file
+echo "## `head -n 1 $1 | sed -e "s;#\s*;\[;g" -e "s;\(.*\);\1\](${filename%.md}.html);g"`"
+
--- a/bin/get_static.sh
+++ b/bin/get_static.sh
@@ -49,4 +49,5 @@ do
    echo $wg
    wget -qO- $baseurl/forschung/$wg > $RESEARCH_BUILD/forschung/$wg.tmp
    iconv -f ISO-8859-1 -t UTF-8 $RESEARCH_BUILD/forschung/$wg.tmp > $RESEARCH_BUILD/forschung/$wg.html
-done
\ No newline at end of file
+done
+
--- a/bin/integrate_pandoc.sh
+++ b/bin/integrate_pandoc.sh
 #!/bin/bash

-RESEARCH_STATIC=$RESEARCH_ROOT/static
-RESEARCH_BUILD=$RESEARCH_STATIC/build
+if [ $# -lt 1 ]
+then
+    RESEARCH_BUILD=$RESEARCH_ROOT/static/build
+    echo -e "No building directory was specified."
+else
+    RESEARCH_BUILD="$1"/build
+fi
+mkdir -p $RESEARCH_BUILD
+echo -e "Building into directory $RESEARCH_BUILD ..."
+

 cd $RESEARCH_BUILD

@@ -23,15 +31,6 @@ do
    -c "css/base_mode.css"
 done

-# get full TUHH staff pages ready for insertion
-rm -rf $RESEARCH_ROOT/static/build/plain
-mkdir -p $RESEARCH_ROOT/static/build/plain
-cp $RESEARCH_ROOT/static/home/*.html $RESEARCH_ROOT/static/build/plain
-cp $RESEARCH_ROOT/static/forschung/*.html $RESEARCH_ROOT/static/build/plain
-
-cd $RESEARCH_ROOT/static/build
-rm -rf *_final.html 
-
 # replace "MARKER" in .html files with research topics
 for f in $RESEARCH_ROOT/static/build/plain/*html
 do
@@ -45,3 +44,4 @@ do
    filename=$(basename -- "$f")
    sed -e "/MARKER/r $RESEARCH_ROOT/static/build/${filename%.md}_research.html" -e '/MARKER/d' $RESEARCH_ROOT/static/topic/topic-template.html > "${filename%.md}"_final.html
 done
+
--- a/bin/preprocIndexMd.sh
+++ b/bin/preprocIndexMd.sh
@@ -54,4 +54,5 @@ do
 done

 # adpat img path (prefix a dot)
-sed -i "s;\](/img/;\](./img/;g" $RESEARCH_BUILD/$filename 
\ No newline at end of file
+sed -i "s;\](/img/;\](./img/;g" $RESEARCH_BUILD/$filename 
+
--- a/bin/preprocMd.sh
+++ b/bin/preprocMd.sh
@@ -65,4 +65,5 @@ do

 # adpat img path (prefix a dot)
 sed -i "s;\](/img/;\](./img/;g" $RESEARCH_BUILD/$filename 
-done
\ No newline at end of file
+done
+
--- a/bin/preproc_staff.sh
+++ b/bin/preproc_staff.sh
@@ -68,4 +68,5 @@ do
 done

 # adpat img path (prefix a dot)
-sed -i "s;\](/img/;\](./img/;g" $RESEARCH_BUILD/$filename 
\ No newline at end of file
+sed -i "s;\](/img/;\](./img/;g" $RESEARCH_BUILD/$filename 
+
--- a/bin/preproc_topics.sh
+++ b/bin/preproc_topics.sh
 #!/bin/bash
 # processing command line arguments
+# Check commandline arguments
 if [ $# -lt 1 ]
 then
-    RESEARCH_BUILD="$RESEARCH_ROOT/build"
+    RESEARCH_BUILD=`pwd`"/build"
+    RESEARCH_TOPICS=`pwd`
    echo -e "No building directory was specified."
+elif [ $# -lt 2 ] 
+then
+    RESEARCH_BUILD="$1"
+    RESEARCH_TOPICS=$RESEARCH_BUILD
+    echo -e "No extra topics directory was specified."
 else
-    RESEARCH_BUILD="$1"/build
+    RESEARCH_BUILD="$1"
+    RESEARCH_TOPICS="$2"
 fi
-echo -e "Building into directory $RESEARCH_BUILD ..."
-rm -rf $RESEARCH_BUILD 
 mkdir -p $RESEARCH_BUILD
+echo -e "Building into directory $RESEARCH_BUILD ..."
+echo -e "Using .md-files from $RESEARCH_TOPICS ..."
+echo -e "Building into directory $RESEARCH_BUILD ..."
+#rm -rf $RESEARCH_BUILD 
+
 baseurl='https://www.mat.tuhh.de'

 #build index.html
 #replace working group
+echo "Preprocessing Working Groups"
 cp $RESEARCH_ROOT/index.md $RESEARCH_BUILD/index.md
 $RESEARCH_ROOT/bin/preproc_wg.sh $RESEARCH_BUILD/index.md $RESEARCH_BUILD >> $RESEARCH_BUILD/build.log 2>&1

 #replace staff names
+echo "Preprocessing Staff"
 $RESEARCH_ROOT/bin/preproc_staff.sh $RESEARCH_BUILD/index.md $RESEARCH_BUILD >> $RESEARCH_BUILD/build.log 2>&1

 #finalize file
-sed -i "s;\](/img/;\](./img/;g" -e '/PROCESSED AUTOMATICALLY/ d' $RESEARCH_BUILD/index.md
+sed -i -e "s;\](/img/;\](./img/;g" -e '/PROCESSED AUTOMATICALLY/ d' $RESEARCH_BUILD/index.md


 #build rest of the topics
-for f in $RESEARCH_ROOT/topics/*.md
+for f in $RESEARCH_TOPICS/*.md
 do
    filename=$(basename -- "$f")
    #make output copy
@@ -49,4 +62,5 @@ do
    sed -i "s;\](/img/;\](./img/;g" $RESEARCH_BUILD/$filename
 done

-echo "Finished preprocessing topics."
\ No newline at end of file
+echo "Finished preprocessing topics."
+
--- a/bin/preproc_wg.sh
+++ b/bin/preproc_wg.sh
@@ -55,4 +55,5 @@ do

    sed -i "s;### Working Groups:\(.*\)$wg\(.*\);### Working Groups:\1\[$fullname\]($wg.html)\2;g" $RESEARCH_BUILD/$filename  
    #sed -i "s;### Working Groups:\(.*\)$wg\(.*\);### Working Groups:\1\[$fullname\]($baseurl/forschung/$wg)\2;g" $RESEARCH_BUILD/$filename  #uncomment this line for linking the mat.tuhh.de webpage
-done
\ No newline at end of file
+done
+
--- a/bin/replace_marker.sh
+++ b/bin/replace_marker.sh
+#!/bin/bash
+# processing command line arguments
+
+# Check commandline arguments
+if [ $# -lt 1 ]
+then
+    RESEARCH_BUILD=`pwd`"/build"
+    RESEARCH_TOPICS=`pwd`
+    echo -e "No building directory was specified."
+elif [ $# -lt 2 ] 
+then
+    RESEARCH_BUILD="$1"
+    RESEARCH_TOPICS=$RESEARCH_BUILD
+    echo -e "No extra topics directory was specified."
+else
+    RESEARCH_BUILD="$1"
+    RESEARCH_TOPICS="$2"
+fi
+mkdir -p $RESEARCH_BUILD
+echo -e "Building into directory $RESEARCH_BUILD ..."
+echo -e "Including _research.html-files from $RESEARCH_TOPICS ..."
+# replace "MARKER" in .html files with research topics
+for f in $RESEARCH_ROOT/static/{home,forschung}/*.html
+do
+    filename=$(basename -- "$f")
+    if [ -f "$RESEARCH_TOPICS/${filename%.html}_research.html" ] 
+    then
+        echo "found $f"
+        sed -e "/MARKER/r  $RESEARCH_TOPICS/${filename%.html}_research.html" \
+            -e '/MARKER/d' $f > "$RESEARCH_BUILD/${filename%.html}"_final.html
+    fi
+
+done
+
+# Building standalone topics and index
+RESEARCH_TEMPLATE="$RESEARCH_ROOT/static/topic/topic-template.html"
+for f in $RESEARCH_ROOT/topics/*.md $RESEARCH_BUILD/index.md
+do
+    filename=$(basename -- "$f")
+    echo "Looking for $RESEARCH_TOPICS/${filename%.md}_research.html"
+    if [ -f "$RESEARCH_TOPICS/${filename%.md}_research.html" ] 
+    then
+        echo "found $f"
+        echo "creating "$RESEARCH_BUILD/${filename%.md}"_final.html"
+        sed -e "/MARKER/r $RESEARCH_TOPICS/${filename%.md}_research.html" \
+            -e '/MARKER/d' $RESEARCH_TEMPLATE > "$RESEARCH_BUILD/${filename%.md}"_final.html
+    fi
+done
+
+# Building standalone topics and index
+#for f in $RESEARCH_ROOT/topics/*.md $RESEARCH_BUILD/index.md
+#do
+#    filename=$(basename -- "$f")
+#    sed -e "/MARKER/r $RESEARCH_ROOT/static/build/${filename%.md}_research.html" -e '/MARKER/d' $RESEARCH_ROOT/static/topic/topic-template.html > "${filename%.md}"_final.html
+#done
+
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -63,4 +63,5 @@ fi
 #END Test 4

 #this should be the last line (errors exit earlier)
-exit 0
\ No newline at end of file
+exit 0
+
--- a/bin/set_environment.sh
+++ b/bin/set_environment.sh
+#!/bin/bash
+# set environment
+# set the environment variables (adapt accordingly)
+
+echo "Setting RESEARCH_ROOT from"
+echo "'" $RESEARCH_ROOT "' to"
+RESEARCH_ROOT=`pwd`; export RESEARCH_ROOT
+echo "'" $RESEARCH_ROOT "'"
+