From 8ecac4c3d670b12b6ff006371a34a569c533a094 Mon Sep 17 00:00:00 2001 From: timurgordon Date: Tue, 6 Sep 2022 23:24:55 +0200 Subject: [PATCH] refactor format_data --- format_data.sh | 357 ++++++++++++++----------------------------------- 1 file changed, 99 insertions(+), 258 deletions(-) diff --git a/format_data.sh b/format_data.sh index 27caa81f0..7c44be126 100644 --- a/format_data.sh +++ b/format_data.sh @@ -1,198 +1,54 @@ #/bin/bash - -dir="threefold_data/content/blog" - -echo "Formatting data..." - -for file in $(grep -ril 'authors:' $dir); -do - echo $file - #FPATH="${file%/*}" - #AUTHOR="$(grep -n 'author:' "$file" | sed -n -e 's/^.*\(\(author:\).*\)/\1/p')" - #LINE1="$(grep -n 'author:' "$file" | head -n1 | sed 's/:.*//')" - #LINE2="$(grep -n 'authorImg:' "$file" | head -n1 | sed 's/:.*//')" - #AUTHOR_NAME="${AUTHOR#* }" - - #echo $AUTHOR_NAME | sed 's/[A-Z]/\L&/g' - #AUTHOR_NAME=$(echo "$AUTHOR_NAME" | tr '[:upper:]' '[:lower:]') - #AUTHOR_SNAKE=${AUTHOR_NAME// /_} - - # move created field below excerpt - DATE="$(grep -n 'created:' "$file" | sed -n -e 's/^.*\(\(created:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'created:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'excerpt:' "$file" | head -n1 | sed 's/:.*//')" - sed -a -i "" "${LINE_NUM}i\\ -$DATE\\ -" "$file" - - # sed -i "" "${LINE1}d" "$file" - - #template: blogPage.html - - # move category field below people - CATEGORY="$(grep -n 'category:' "$file" | sed -n -e 's/^.*\(\(category:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'category:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'authors:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $CATEGORY\\ - " "$file" - - # move tags field below people - TAGS="$(grep -n 'tags:' "$file" | sed -n -e 's/^.*\(\(tags:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'tags:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'authors:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $TAGS\\ -" "$file" - - # move image field below category - IMAGE="$(grep -n 'image:' "$file" | sed -n -e 's/^.*\(\(image:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'image:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'category:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ -$IMAGE\\ -" "$file" - - # Change field names - sed -i "" "s|category:|categories:|g" "$file" - sed -i "" "s|created:|date:|g" "$file" - sed -i "" "s|excerpt:|description:|g" "$file" - sed -i "" "s| ---|---|g" "$file" - sed -i "" "s|authors:|taxonomies:\\ - people:|g" "$file" - sed -i "" "s|image: ./|extra:\\ - imgPath: |g" "$file" - - mv $file ${file%/*}/index.md - mv ${file%/*} content/blog - - #mv 'static/'$IMG_PATH $FPATH - - #="$(grep -n 'imgPath:' "$file")" - #ECHO $NUM - #IMG_PATH="$(echo $NUM | sed -n -e 's/^.*\(\(images\/blog\/\).*\)/\1/p')" - #echo $IMG_PATH - #IMG_NAME="${IMG_PATH##*/}" - #echo $IMG_NAME - #mv 'static/'$IMG_PATH $FPATH - #sed -i "" "s|$IMG_PATH|$IMG_NAME|g" "$file" -done; - -dir="threefold_data/content/news" - -for file in $(grep -ril 'authors:' $dir); -do - echo $file - #FPATH="${file%/*}" - #AUTHOR="$(grep -n 'author:' "$file" | sed -n -e 's/^.*\(\(author:\).*\)/\1/p')" - #LINE1="$(grep -n 'author:' "$file" | head -n1 | sed 's/:.*//')" - #LINE2="$(grep -n 'authorImg:' "$file" | head -n1 | sed 's/:.*//')" - #AUTHOR_NAME="${AUTHOR#* }" - - #echo $AUTHOR_NAME | sed 's/[A-Z]/\L&/g' - #AUTHOR_NAME=$(echo "$AUTHOR_NAME" | tr '[:upper:]' '[:lower:]') - #AUTHOR_SNAKE=${AUTHOR_NAME// /_} - - # move created field below excerpt - DATE="$(grep -n 'created:' "$file" | sed -n -e 's/^.*\(\(created:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'created:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'excerpt:' "$file" | head -n1 | sed 's/:.*//')" - sed -a -i "" "${LINE_NUM}i\\ -$DATE\\ -" "$file" - - # sed -i "" "${LINE1}d" "$file" - - #template: blogPage.html - - # move category field below people - CATEGORY="$(grep -n 'category:' "$file" | sed -n -e 's/^.*\(\(category:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'category:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'authors:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $CATEGORY\\ - " "$file" - - # move tags field below people - TAGS="$(grep -n 'tags:' "$file" | sed -n -e 's/^.*\(\(tags:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'tags:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'authors:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $TAGS\\ -" "$file" - - # move image field below category - IMAGE="$(grep -n 'image:' "$file" | sed -n -e 's/^.*\(\(image:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'image:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'category:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ -$IMAGE\\ -" "$file" - - # Change field names - sed -i "" "s|category:|categories:|g" "$file" - sed -i "" "s|created:|date:|g" "$file" - sed -i "" "s|excerpt:|description:|g" "$file" - sed -i "" "s| ---|---|g" "$file" - sed -i "" "s|authors:|taxonomies:\\ - people:|g" "$file" - sed -i "" "s|image: ./|extra:\\ - imgPath: |g" "$file" - - mv $file ${file%/*}/index.md - mv ${file%/*} content/newsroom - - #mv 'static/'$IMG_PATH $FPATH - - #="$(grep -n 'imgPath:' "$file")" - #ECHO $NUM - #IMG_PATH="$(echo $NUM | sed -n -e 's/^.*\(\(images\/blog\/\).*\)/\1/p')" - #echo $IMG_PATH - #IMG_NAME="${IMG_PATH##*/}" - #echo $IMG_NAME - #mv 'static/'$IMG_PATH $FPATH - #sed -i "" "s|$IMG_PATH|$IMG_NAME|g" "$file" - -done; - +echo "Running threefold_data formatter:" +echo "Formatting people..." dir="threefold_data/content/person" + +# function that checks whether given file is formatted +# simply compares if keyword excerpt or description is used +# $1: file + +exists() { + if grep "$1" $2 > /dev/null + then + return 0 + else + return 1 + fi +} + +move_below() { + if exists $1 $3; then + TARGET="$(grep -n "$1" "$3" | sed -n -e "s/^.*\(\($1\).*\)/\1/p")" + LINE_NUM="$(grep -n "$1" "$3" | head -n1 | sed 's/:.*//')" + sed -i "" "${LINE_NUM}d" "$3" + LINE_NUM="$(grep -n "$2" "$3" | head -n1 | sed 's/:.*//')" + LINE_NUM="$(($LINE_NUM + 1))" + sed -a -i "" "${LINE_NUM}i\\ +$TARGET\\ +" "$3" + fi +} + +replace() { + sed -i "" "s|$1|$2|g" "$file" +} for file in $(grep -ril 'name:' $dir); do echo $file - # move created field below excerpt - #BIO="$(grep -n 'bio:' "$file" | sed -n -e 's/^.*\(\(bio:\).*\)/\1/p')" - #LINE_NUM="$(grep -n 'bio:' "$file" | head -n1 | sed 's/:.*//')" - #sed -i "" "${LINE_NUM}d" "$file" - #LINE_NUM="$(grep -n 'socialLinks:' "$file" | head -n1 | sed 's/:.*//')" - #sed -a -i "" "${LINE_NUM+}i\\ -#$DATE\\ -#" "$file" - - sed -i "" "s|private: 0| private: 0\\ - socialLinks: {\\ - }|g" "$file" - sed -i "" "s|private: 1| private: 1\\ socialLinks: {\\ }|g" "$file" + move_below category: excerpt: $file + move_below memberships: excerpt: $file + move_below websites: socialLinks: $file + move_below projects: image: $file + move_below linkedin: socialLinks: $file + move_below github: linkedin: $file + # move bio field below toml BIO="$(grep -n 'bio:' "$file" | sed -n -e 's/^.*\(\(bio:\).*\)/\1/p')" LINE_NUM="$(grep -n 'bio:' "$file" | head -n1 | sed 's/:.*//')" @@ -200,71 +56,6 @@ do echo "\n$BIO" >> $file sed -i "" "s|bio: ||g" "$file" - # move category field below excerpt - CATEGORY="$(grep -n 'category:' "$file" | sed -n -e 's/^.*\(\(category:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'category:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'excerpt:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $CATEGORY\\ - " "$file" - - # move memberships field below excerpt - MEMBERSHIPS="$(grep -n 'memberships:' "$file" | sed -n -e 's/^.*\(\(memberships:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'memberships:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'excerpt:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $MEMBERSHIPS\\ -" "$file" - - # move website field below socialLinks - MEMBERSHIPS="$(grep -n 'websites:' "$file" | sed -n -e 's/^.*\(\(websites:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'websites:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'socialLinks:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $MEMBERSHIPS\\ -" "$file" - - # move project field below socialLinks - MEMBERSHIPS="$(grep -n 'projects:' "$file" | sed -n -e 's/^.*\(\(projects:\).*\)/\1/p')" - LINE_NUM="$(grep -n 'projects:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'image:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $MEMBERSHIPS\\ -" "$file" - - LINKEDIN="$(grep -n 'linkedin:' "$file" | sed -n -e 's/^.*\(\(linkedin:\).*\)/\1/p')" - if [ "$LINKEDIN" != "" ]; then - LINE_NUM="$(grep -n 'linkedin:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'socialLinks:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $LINKEDIN\\ -" "$file" - sed -i "" "s|$LINKEDIN|$LINKEDIN,|g" "$file" - fi - - GITHUB="$(grep -n 'github:' "$file" | sed -n -e 's/^.*\(\(github:\).*\)/\1/p')" - if [ "$GITHUB" != "" ]; then - LINE_NUM="$(grep -n 'github:' "$file" | head -n1 | sed 's/:.*//')" - sed -i "" "${LINE_NUM}d" "$file" - LINE_NUM="$(grep -n 'linkedin:' "$file" | head -n1 | sed 's/:.*//')" - LINE_NUM="$(($LINE_NUM + 1))" - sed -a -i "" "${LINE_NUM}i\\ - $GITHUB\\ - " "$file" - sed -i "" "s|$GITHUB|$GITHUB,|g" "$file" - fi - - WEBSITES="$(grep -n 'websites:' "$file" | sed -n -e 's/^.*\(\(websites:\).*\)/\1/p')" sed -i "" "s|$WEBSITES|$WEBSITES,|g" "$file" @@ -282,6 +73,7 @@ do id: ${NAME}\\ " "$file" + # Change field names sed -i "" "s|category:|categories:|g" "$file" sed -i "" "s|rank:|weight:|g" "$file" @@ -304,15 +96,64 @@ id: ${NAME}\\ mv $file ${file%/*}/index.md mv ${file%/*} content/people - #mv 'static/'$IMG_PATH $FPATH - - #="$(grep -n 'imgPath:' "$file")" - #ECHO $NUM - #IMG_PATH="$(echo $NUM | sed -n -e 's/^.*\(\(images\/blog\/\).*\)/\1/p')" - #echo $IMG_PATH - #IMG_NAME="${IMG_PATH##*/}" - #echo $IMG_NAME - #mv 'static/'$IMG_PATH $FPATH - #sed -i "" "s|$IMG_PATH|$IMG_NAME|g" "$file" + break done; + +echo "Formatting blogs..." +dir="threefold_data/content/blog" + +for file in $(grep -ril 'authors:' $dir); +do + echo $file + + move_below created: excerpt: $file + move_below category: authors: $file + move_below tags: authors: $file + move_below image: category: $file + + # Change field names + sed -i "" "s|category:|categories:|g" "$file" + sed -i "" "s|created:|date:|g" "$file" + sed -i "" "s|excerpt:|description:|g" "$file" + sed -i "" "s| ---|---|g" "$file" + sed -i "" "s|authors:|taxonomies:\\ + people:|g" "$file" + sed -i "" "s|image: ./|extra:\\ + imgPath: |g" "$file" + + mv $file ${file%/*}/index.md + mv ${file%/*} content/blog + + break +done; + +echo "Formatting news..." +dir="threefold_data/content/news" + +for file in $(grep -ril 'authors:' $dir); +do + echo $file + + move_below created: excerpt: $file + move_below category: authors: $file + move_below tags: authors: $file + move_below image: category: $file + + # Change field names + sed -i "" "s|category:|categories:|g" "$file" + sed -i "" "s|created:|date:|g" "$file" + sed -i "" "s|excerpt:|description:|g" "$file" + sed -i "" "s| ---|---|g" "$file" + sed -i "" "s|authors:|taxonomies:\\ + people:|g" "$file" + sed -i "" "s|image: ./|extra:\\ + imgPath: |g" "$file" + + mv $file ${file%/*}/index.md + mv ${file%/*} content/newsroom + + break + +done; +