WebSVN – SmartDukaan – /trunk/runutils/generate-content-from-catalog.sh

#!/bin/bash

# resolve links - $0 may be a softlink
PRG="$0"

while [ -h "$PRG" ]; do
  ls=`ls -ld "$PRG"`
  link=`expr "$ls" : '.*-> \(.*\)$'`
  if expr "$link" : '/.*' > /dev/null; then
    PRG="$link"
  else
    PRG=`dirname "$PRG"`/"$link"
  fi
done

# Get standard environment variables
PRGDIR=`dirname $(readlink -f "$PRG")`

echo 1 > $PRGDIR/status

set -e

PROJECT_DIR="`dirname ${PRGDIR}`"
LOG_DIR="/var/log/content-generation"
DATE=`date +%Y-%b-%d-%Hh%Mm%Ss`
MYSQL_DUMP="${PRGDIR}/partial-catalog-${DATE}.sql"
CONTENT_LOG="${LOG_DIR}/content-generation-${DATE}.log"
COMPARISON_LOG="${LOG_DIR}/comparison-score-computation-${DATE}.log"
PRODUCT_LIST_LOG="${LOG_DIR}/product-list-${DATE}.log"
HELPDOCS_LOG="${LOG_DIR}/helpdocs-generation-${DATE}.log"
IR_LOG="${LOG_DIR}/irdata-generation-${DATE}.log"
SOLR_IRDATA=/var/lib/tomcat6/webapps/export/solr/

STAGING_SERVER="192.168.153.127"
STAGING_ENTITIES_EXPORT_PATH="entities-shop2020"

STORE_SERVER="192.168.177.42"
STORE_ENTITIES_EXPORT_PATH="entities-store"

PROD_SERVERS="192.168.166.159 192.168.161.5 109.74.200.220"
PROD_ENTITIES_EXPORT_PATH="entities-saholic"

STATIC_SERVERS="173.230.151.179 178.79.184.36"
SUPPORT_SERVER="192.168.149.148"

ENTITIES_DEST_PATH="/var/lib/tomcat6/webapps/export/html/entities.tgz"

MAVEN_REPO="${HOME}/.m2/repository"
M2_CLASSPATH="${MAVEN_REPO}/in/shop2020/ContentStore/1.0-SNAPSHOT/ContentStore-1.0-SNAPSHOT.jar:${MAVEN_REPO}/in/shop2020/ThriftConfig/1.0-SNAPSHOT/ThriftConfig-1.0-SNAPSHOT.jar:${MAVEN_REPO}/in/shop2020/Common/1.0-SNAPSHOT/Common-1.0-SNAPSHOT.jar:${MAVEN_REPO}/readonly/apache/velocity/1.6.3/velocity-1.6.3.jar:${MAVEN_REPO}/readonly/apache/commons-lang/2.4/commons-lang-2.4.jar:${MAVEN_REPO}/readonly/apache/commons-collections/3.2.1/commons-collections-3.2.1.jar:${MAVEN_REPO}/org/apache/thrift/libthrift/0.7.0/libthrift-0.7.0.jar:${MAVEN_REPO}/readonly/slf4j/slf4j-api/1.5.8/slf4j-api-1.5.8.jar:${MAVEN_REPO}/readonly/slf4j/slf4j-simple/1.5.8/slf4j-simple-1.5.8.jar:${MAVEN_REPO}/readonly/apache/commons-logging-api/1.1/commons-logging-api-1.1.jar:${MAVEN_REPO}/readonly/bdb/je/4.1.7/je-4.1.7.jar:${MAVEN_REPO}/readonly/jython/2.5.1/jython-2.5.1.jar:${MAVEN_REPO}/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:${MAVEN_REPO}/readonly/apache/commons-io/1.3.2/commons-io-1.3.2.jar:${MAVEN_REPO}/org/json/json/20090211/json-20090211.jar:${MAVEN_REPO}/org/apache/velocity/velocity-tools/2.0/velocity-tools-2.0.jar:${MAVEN_REPO}/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:${MAVEN_REPO}/javax/mail/mail/1.4/mail-1.4.jar:${MAVEN_REPO}/readonly/apache/poi-3.6/20091214/poi-3.6-20091214.jar:${MAVEN_REPO}/readonly/apache/poi-scratchpad-3.6/20091214/poi-scratchpad-3.6-20091214.jar:${MAVEN_REPO}/commons-net/commons-net/3.0.1/commons-net-3.0.1.jar:${MAVEN_REPO}/junit/junit/4.8.2/junit-4.8.2.jar:${MAVEN_REPO}/org/mongodb/mongo-java-driver/2.11.0/mongo-java-driver-2.11.0.jar"

echo "Generating content for one entities ${PROJECT_DIR}"
echo "======================================================="
cd ${PROJECT_DIR}/ContentStore
java -Xms256m -Xmx512m -cp ${M2_CLASSPATH}  in.shop2020.util.ContentGenerationUtility -u CATALOG -t ALL > ${CONTENT_LOG} 
echo "======================================================="
echo "Content generation completed"

echo "Dumping item, vendoritemmapping and vendoritempricing tables to be imported on staging and production servers"
echo "======================================================="
cd ${PRGDIR}
mysqldump -uroot -pshop2020 catalog --tables item similaritems source sourceitempricing category voucheritemmapping freebieitem entitytag amazonlisted storepricing brandinfo iteminsurermapping > ${MYSQL_DUMP}

#get dump for siteContent
mongodump -d CONTENT -c siteContent

echo "======================================================"
echo " Validating the generated-content."
echo "======================================================"
${PRGDIR}/catalog-content-validation.sh
#if [ $? -ne 0 ]; then
#       exit 1
#fi
echo "Rsyncing generated content to staging and production servers"
echo "======================================================="
#Update media content on staging, store and production
cd /var/lib/tomcat6/webapps/export
rsync -az --delete media/ ${STAGING_SERVER}:/var/lib/tomcat6/webapps/export/media-offline
cd /var/lib/tomcat6/webapps/export
rsync -az --delete media/ ${STORE_SERVER}:/var/lib/tomcat6/webapps/export/media-offline
cd /var/lib/tomcat6/webapps/export/media
for PROD_SERVER in ${PROD_SERVERS}
do
        rsync -az --delete website ${PROD_SERVER}:/var/lib/tomcat6/webapps/export/media-offline
done

for STATIC_SERVER in ${STATIC_SERVERS}
do
        rsync -az --delete static ${STATIC_SERVER}:/var/www/media-offline
done

cd /var/lib/tomcat6/webapps/export

rm -f javascripts.tgz
mv javascripts javascripts-${DATE}; tar -zcf javascripts.tgz javascripts-${DATE}; mv javascripts-${DATE} javascripts
scp javascripts.tgz ${STAGING_SERVER}:/var/lib/tomcat6/webapps/export/
scp javascripts.tgz ${STORE_SERVER}:/var/lib/tomcat6/webapps/export/
for PROD_SERVER in ${PROD_SERVERS}
do
        scp javascripts.tgz ${PROD_SERVER}:/var/lib/tomcat6/webapps/export/
done

for STATIC_SERVER in ${STATIC_SERVERS}
do
        scp javascripts.tgz ${STATIC_SERVER}:/var/www/
done

rsync -az --delete documents/ ${STAGING_SERVER}:/var/lib/tomcat6/webapps/export/documents-offline
rsync -az --delete documents/ ${STORE_SERVER}:/var/lib/tomcat6/webapps/export/documents-offline

for STATIC_SERVER in ${STATIC_SERVERS}
do
        rsync -az --delete documents/ ${STATIC_SERVER}:/var/www/documents-offline
done

#Update entities on staging using the entities-shop2020 direcotry
#Update entities on store using the entities-store direcotry
#Update entities on production using the entities-saholic directory
cd /var/lib/tomcat6/webapps/export/html
rm -f ${STAGING_ENTITIES_EXPORT_PATH}.tgz ${PROD_ENTITIES_EXPORT_PATH}.tgz ${STORE_ENTITIES_EXPORT_PATH}.tgz
mv ${STAGING_ENTITIES_EXPORT_PATH} entities-${DATE}
tar -zcf ${STAGING_ENTITIES_EXPORT_PATH}.tgz entities-${DATE}
mv entities-${DATE} ${STAGING_ENTITIES_EXPORT_PATH}

mv ${STORE_ENTITIES_EXPORT_PATH} entities-${DATE}
tar -zcf ${STORE_ENTITIES_EXPORT_PATH}.tgz entities-${DATE}
mv entities-${DATE} ${STORE_ENTITIES_EXPORT_PATH}

mv ${PROD_ENTITIES_EXPORT_PATH} entities-${DATE}
tar -zcf ${PROD_ENTITIES_EXPORT_PATH}.tgz entities-${DATE}
mv entities-${DATE} ${PROD_ENTITIES_EXPORT_PATH}

scp ${STAGING_ENTITIES_EXPORT_PATH}.tgz ${STAGING_SERVER}:${ENTITIES_DEST_PATH}
scp ${STORE_ENTITIES_EXPORT_PATH}.tgz ${STORE_SERVER}:${ENTITIES_DEST_PATH}
for PROD_SERVER in ${PROD_SERVERS}
do
        scp ${PROD_ENTITIES_EXPORT_PATH}.tgz ${PROD_SERVER}:${ENTITIES_DEST_PATH}
done

#Update helpdocs on production and staging
cd /var/lib/tomcat6/webapps/export/html
rm -f helpdocs.tgz
mv helpdocs helpdocs-${DATE}; tar -zcf helpdocs.tgz helpdocs-${DATE}; mv helpdocs-${DATE} helpdocs
scp helpdocs.tgz ${STAGING_SERVER}:/var/lib/tomcat6/webapps/export/
scp helpdocs.tgz ${STORE_SERVER}:/var/lib/tomcat6/webapps/export/
for PROD_SERVER in ${PROD_SERVERS}
do
        scp helpdocs.tgz ${PROD_SERVER}:/var/lib/tomcat6/webapps/export/
done

#Update mongo for mobile website to prod1 and shop2020
cd ${PRGDIR}
rm -f dump.tgz
mv dump dump-${DATE}; tar -zcf dump.tgz dump-${DATE}; mv dump-${DATE} dump
scp dump.tgz ${STAGING_SERVER}:/var/lib/tomcat6/webapps/export/
for PROD_SERVER in ${PROD_SERVERS}
do
        scp dump.tgz ${PROD_SERVER}:/var/lib/tomcat6/webapps/export/
done


#Update product list data on staging and production servers
cd /var/lib/tomcat6/webapps/export
rm -f partners.tgz
mv partners partners-${DATE}; tar -zcf partners.tgz partners-${DATE}; mv partners-${DATE} partners
scp partners.tgz ${STAGING_SERVER}:/var/lib/tomcat6/webapps/export/
for PROD_SERVER in ${PROD_SERVERS}
do
        scp partners.tgz ${PROD_SERVER}:/var/lib/tomcat6/webapps/export/
done

for STATIC_SERVER in ${STATIC_SERVERS}
do
        scp partners.tgz ${STATIC_SERVER}:/var/www
done

#Update solr data on staging, store and production servers
cd /var/lib/tomcat6/webapps/export
rm -f solr.tgz
mv solr solr-${DATE}; tar -zcf solr.tgz solr-${DATE}; mv solr-${DATE} solr
scp solr.tgz ${STAGING_SERVER}:/var/lib/tomcat6/webapps/export/
scp solr.tgz ${STORE_SERVER}:/var/lib/tomcat6/webapps/export/
for PROD_SERVER in ${PROD_SERVERS}
do
        scp solr.tgz ${PROD_SERVER}:/var/lib/tomcat6/webapps/export/
done

#Put mysql dump on staging and production servers
scp ${MYSQL_DUMP} ${STAGING_SERVER}:/catalog-dumps
scp ${MYSQL_DUMP} ${SUPPORT_SERVER}:/catalog-dumps
for PROD_SERVER in ${PROD_SERVERS}
do
        scp ${MYSQL_DUMP} ${PROD_SERVER}:/catalog-dumps
done

ssh ${STAGING_SERVER} "echo ${DATE}>/catalog-dumps/latest.timestamp"
ssh ${STORE_SERVER} "echo ${DATE}>/catalog-dumps/latest.timestamp"
ssh ${SUPPORT_SERVER} "echo ${DATE}>/catalog-dumps/latest.timestamp"
for PROD_SERVER in ${PROD_SERVERS}
do
        ssh ${PROD_SERVER} "echo ${DATE}>/catalog-dumps/latest.timestamp"
done

for STATIC_SERVER in ${STATIC_SERVERS}
do
        ssh ${STATIC_SERVER} "echo ${DATE}>/catalog-dumps/latest.timestamp"
done

echo "Done copying content to staging and production servers"
echo "======================================================="
echo 0 > $PRGDIR/status
Subversion Repositories SmartDukaan

(root)/trunk/runutils/generate-content-from-catalog.sh – Rev 11035
