PYTHON 121
Digireminder Source v0.8 (Redacted) By theneedyguy on 14th May 2017 08:08:21 PM
  1. """
  2. Prerequisites:
  3. We need to make some changes to the service.
  4. I added a webrequest that is immediately made after restart to trigger @app.before_first_request
  5. Also the lockfile gets reset each time the server gets reset.
  6. File = /etc/init.d/apache2
  7.  
  8. """
  9.  
  10. from flask import Flask
  11. from flask import request
  12. from flask import render_template
  13. from flask import Markup
  14. from email.mime.text import MIMEText
  15. import smtplib
  16. import urllib.parse
  17. import re
  18. from time import sleep
  19. from bs4 import BeautifulSoup
  20. import urllib.request
  21. import threading
  22. import peewee
  23. from peewee import *
  24.  
  25. """
  26. GLOBAL VARIABLES
  27. """
  28. #Option for logging DEV is for development, PROD is for stable
  29. DEVOPTION = "[DEV]"
  30. #Version variable
  31. VERSION = "0.8-Beta-DEV-STAGING"
  32. #Environment
  33. ENV = "digireminder-t"
  34. #SSL used?
  35. SSL = "http://"
  36. #Database
  37. DB = ""
  38. #Website title
  39. TITLE = "Digireminder-Staging"
  40. # Form that is shown, if digitec is available.
  41. FORM_ENABLED = Markup("""<form action="." method="POST">
  42.         <div class="form-group">
  43.                 <label for="inputlg-alias">Produktlink</label>
  44.                 <input class="form-control input-lg" id="inputlg-alias" type="text" name="text" required>
  45.         </div> 
  46.        <div class="form-group">
  47.                <label for="inputlg-secret">Email</label>
  48.                <input class="form-control input-lg" id="inputlg-secret" type="text" name="email" required>
  49.         </div>
  50.        <input type="submit" class="btn btn-primary btn-lg"  name="my-form" value="Remind me!">
  51.    </form>""")
  52. # Form that is shown, if digitec is not available.
  53. FORM_DISABLED = Markup("""
  54.         <div class="card card-inverse card-danger text-center">
  55.           <div class="card-block">
  56.             <blockquote class="card-blockquote">
  57.               <p>Digitec.ch ist nicht erreichbar. Versuche es später wieder.</p>
  58.             </blockquote>
  59.           </div>
  60.         </div>
  61.         <br/>
  62.         <form action="." method="POST">
  63.         <fieldset disabled>
  64.         <div class="form-group">
  65.                 <label for="inputlg-alias">Produktlink</label>
  66.                 <input class="form-control input-lg" id="inputlg-alias" type="text" name="text" required>
  67.         </div> 
  68.        <div class="form-group">
  69.                <label for="inputlg-secret">Email</label>
  70.                <input class="form-control input-lg" id="inputlg-secret" type="text" name="email" required>
  71.         </div>
  72.        <input type="submit" class="btn btn-primary btn-lg"  name="my-form" value="Remind me!">
  73.        </fieldset>
  74.    </form>""")
  75.  
  76.  
  77.  
  78. #This variable defines the db we want to write into
  79. db = MySQLDatabase(host='localhost', user='user',passwd='password', database=DB)
  80.  
  81. """
  82. We use peewee so we don't have to make queries.
  83. This model makes coding a lot easier.
  84. """
  85. class Job(peewee.Model):
  86.         link = peewee.CharField()
  87.         email = peewee.CharField()
  88.  
  89.         class Meta:
  90.                 indexes = (
  91.                         (( 'link', 'email'), True),
  92.                 )
  93.                 database = db
  94.  
  95. app = Flask(__name__)
  96.  
  97. #This will be executed first.
  98. @app.before_first_request
  99. def activate_job():
  100.         #v0.8 try to create table in db.
  101.         try:
  102.                 db.get_conn()
  103.                 Job.create_table()
  104.                 db.close()
  105.         except:
  106.                 pass
  107.         #Define path of lock file.
  108.         filename = "dir"
  109.  
  110.         """
  111.         This def will run in the background to restart jobs that have been cancelled by a server event in background.
  112.         It connects to the Database, collects each existent job and spawns a thread for each job using mail and link as params.
  113.         We also pass a flag so no duplicate job gets created in the database.
  114.         """
  115.         def run_job():
  116.                 print(DEVOPTION +" ------Restarting old tasks...")
  117.                 db.get_conn()
  118.                 for job in Job:
  119.                         #Delay 5 seconds for stability.
  120.                         sleep(5)
  121.                         getAvailabilityThread = threading.Thread(target=timerThread, args=(job.email, job.link, False))
  122.                         getAvailabilityThread.start()
  123.                 db.close()
  124.  
  125.         """
  126.         In activate_job, we open a pseudo lockfile. If it says locked then a thread is spawned to start old tasks.
  127.         Also it writes another value to the file so the next request doesn't spawn another thread to start old tasks.
  128.         """
  129.         print(DEVOPTION +" ------Checking if thread has been executed...")
  130.         with open(filename, "r+") as f:
  131.                 filedata = f.read().replace('\n', '')
  132.                 if filedata == "locked":
  133.                         print(DEVOPTION +" ------Writing to file...")
  134.                         thread = threading.Thread(target=run_job)
  135.                         thread.start()
  136.                         f.seek(0)
  137.                         f.write("unlocked")
  138.                         f.truncate()
  139.  
  140.        
  141.  
  142.  
  143. """
  144. This def just sends a mail using receiver and productlink as parameters.
  145. """
  146. def sendmail(receiver, productlink):
  147.         server = smtplib.SMTP("purpl3.net", 587)
  148.         server.ehlo()
  149.         server.starttls()
  150.         server.login("user", "pw")
  151.         recipients = [receiver]
  152.         sender = "no-reply@purpl3.net"
  153.         subject = "Produkt bei digitec.ch ist nun verfügbar! | Digireminder " + VERSION
  154.         body = '<html><body>'"Hallo, du erhälst dieses Mail, weil du auf https://digireminder.purpl3.net einen Reminder für ein Produkt erstellt hast.<br/>Das Produkt ist jetzt oder wieder an Lager: "+productlink+"<br/><br/>Mit freundlichen Grüssen<br/>Purpl3.net"'</body></html>'
  155.         msg = MIMEText(body.encode('utf-8'), 'html', 'UTF-8')
  156.         msg['Subject'] = subject
  157.         msg['From'] = sender
  158.         msg['Content-Type'] = "text/html; charset=utf-8"
  159.         msg['To'] = ", ".join(recipients)
  160.         try:
  161.                 server.sendmail("no-reply@purpl3.net", receiver, str(msg))
  162.         except:
  163.                 return
  164.                 server.quit()
  165.  
  166. """
  167. This function first connects to the db to create a job using receiver and productlink as params
  168. The param createNewJob just sets a flag if a new job need to be created. (This needs to be there so that old jobs dont create duplicate entries in the db)
  169. After creating the job it starts a webrequest to digitec.ch to check whether the provided product is available
  170. If it is available the job inside the database gets deleted and the mail is sent out to the requestor.
  171.  
  172. """
  173. def getAvailability(receiver, productlink, createNewJob):
  174.         #Define headers and request
  175.         header = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)'}
  176.         req = urllib.request.Request(productlink, headers=header)
  177.         #Create job in database
  178.         if createNewJob == True and urllib.request.urlopen(req).getcode() == 200:
  179.                 print(DEVOPTION + " ------New request found!")
  180.                 db.get_conn()
  181.                 try:
  182.                         job = Job(link=productlink, email=receiver)
  183.                         job.save()
  184.                 except:
  185.                         print(DEVOPTION +" ------Error creating job in database!")
  186.                 db.close()
  187.  
  188.         #Make request and check if available
  189.         try:
  190.                 with urllib.request.urlopen(req) as url:
  191.                         s = url.read()
  192.                         soup = BeautifulSoup(s,"html.parser")
  193.                         avail_div = soup.find("div", id="Availability")
  194.                         svg = avail_div.find("svg", class_="digicon digicon--extra digicon--green")
  195.                         #If available, delete job from database and send mail
  196.                         if svg != None:
  197.                                 print(DEVOPTION +" ------Available!")
  198.                                 #Clean up job from database
  199.                                 db.get_conn()
  200.                                 try:
  201.                                         for job in Job:
  202.                                                 if job.link == productlink and job.email == receiver:
  203.                                                         job.delete_instance()
  204.                                 except:
  205.                                         print(DEVOPTION +" ------Error deleting job in database!")
  206.                                 db.close()
  207.                                 sendmail(receiver, productlink)
  208.                                 return True
  209.                         else:
  210.                                 return False
  211.         except:
  212.                 print(DEVOPTION + " ------Website not found!")
  213.  
  214.  
  215. """
  216. This is the def that spawns the getAvailability def in the background with email and link as params
  217. It will check every 3 hours for a change on the website.
  218. """
  219. def timerThread(receiver, productlink, createNewJob):
  220.         print(DEVOPTION +" ------Started new thread!")
  221.         while getAvailability(receiver, productlink, createNewJob) == False:
  222.                 print(DEVOPTION +" ------Checking for availability...")
  223.                 sleep(10800)
  224.         print(DEVOPTION +" ------Finished thread")
  225.  
  226.  
  227. """
  228. The default route. It loads the startpage. Bevore it makes a chekc to see if digitec is available and loads the corresponding html form.
  229. If digitec is not available then the disabled form will be loaded. Otherwise the normal form loads.
  230. """
  231. @app.route('/')
  232. def my_form():
  233.         header = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)'}
  234.         req = urllib.request.Request("https://www.digitec.ch", headers=header)
  235.         try:
  236.                 if urllib.request.urlopen(req).getcode() != 200:
  237.                         return render_template("my-form.html", version = VERSION, title = TITLE, form = FORM_DISABLED)
  238.                 else:
  239.                         return render_template("my-form.html", version = VERSION, title = TITLE, form = FORM_ENABLED)
  240.         except:
  241.                 print(DEVOPTION + " ------Digitec is not available right now.")
  242.                 return render_template("my-form.html", version = VERSION, title = TITLE, form = FORM_DISABLED)
  243.         #return render_template("my-form.html", version = VERSION, title = TITLE, form = FORM)
  244.  
  245. """
  246. The post method for the default route.
  247. This reads the arguments provided by the form the user sends and puts them into variables.
  248. Then it performs some checks:
  249.         It checks first if the job is already in the database - if not it returns JOB EXISTS
  250.         If no job exists, check if the link is from digitec.ch - if it is then spawn timerThread with email and link and show the user some feedback by loading success.html
  251.  
  252. """
  253. @app.route('/', methods=['POST'])
  254. def my_form_post():
  255.         #v0.8 strip whitespace from string to prevent multiple entries for same item
  256.         email = request.form['email'].strip().replace(" ", "")
  257.         text = request.form['text'].strip().replace(" ", "")
  258.         #Check if exists in DB
  259.         db.get_conn()
  260.         try:
  261.                 for job in Job.filter(email=email):
  262.                         if job.link == text and job.email == email:
  263.                                 return render_template("duplicate.html",  title = TITLE)
  264.         except:
  265.                 print(DEVOPTION +" ------Error! Something went wrong in the POST method.")
  266.         db.close()
  267.  
  268.         if urllib.parse.urlparse(text).hostname == "www.digitec.ch" and re.search("/product/", text) != None and re.search("(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", email) != None:
  269.                 getAvailabilityThread = threading.Thread(target=timerThread, args=(email, text, True))
  270.                 getAvailabilityThread.start()
  271.                 return render_template("success.html", title = TITLE)
  272.         else:
  273.                 print(DEVOPTION +" ------Wrong URL or Email entered!")
  274.                 return render_template("error_hostname.html",  title = TITLE)
  275.  
  276.  
  277. @app.route('/changelog')
  278. def changelog():
  279.         return render_template("changelog.html", title = TITLE)
  280. """
  281. This def reacts to requests and starts the start_loop if it detects a request.
  282. start_loop is programmed to only take the first request as a start signal using a mechanism using a lock file.
  283. """
  284. def start_runner():
  285.         def start_loop():
  286.                 not_started = True
  287.                 while not_started:
  288.                         try:
  289.                                 #This will initiate the first job before first "official" request.
  290.                                 r = urllib.request.urlopen(SSL+ENV+".purpl3.net/")
  291.                                 if r.getcode() == 200:
  292.                                         print('Server started, quiting start_loop')
  293.                                         not_started = False
  294.                         except:
  295.                                 print('Server not yet started')
  296.                         sleep(2)
  297.  
  298.         thread = threading.Thread(target=start_loop)
  299.         thread.start()
  300.  
  301.  
  302.  
  303. if __name__ == '__main__':
  304.         app.run()

Paste is for source code and general debugging text.

Login or Register to edit, delete and keep track of your pastes and more.

Raw Paste

Login or Register to edit or fork this paste. It's free.