Archive
An SQLite example
Here you will find a very basic program that stores data in SQLite. It is a simple example. If I need to write a program that needs sqlite, it can serve as a basis and I won’t need to write it from scratch.
Problem
Visit the page /r/earthporn, extract the image URLs and save the URLs of images stored on imgur in an sqlite table.
Solution
earthporn.py:
#!/usr/bin/env python
# encoding: utf-8
import requests
import database as db # see below
subreddit = "earthporn"
def main():
db.init() # Important! It will connect to the DB.
r = requests.get('http://www.reddit.com/r/{sr}/.json'.format(sr=subreddit))
d = r.json()
children = d["data"]["children"]
for e in children:
url = e["data"]["url"]
if "imgur.com" in url:
# print url
db.add_image(url, subreddit)
####################
if __name__ == "__main__":
main()
database.py:
#!/usr/bin/env python
"""
Sqlite database handler.
"""
import os
import sqlite3
import atexit
import random
import termcolor
PATH = os.path.dirname(os.path.abspath(__file__))
SCHEMA = """
CREATE TABLE "images" (
"url" TEXT PRIMARY KEY NOT NULL,
"subreddit" TEXT,
"insert_date" DEFAULT CURRENT_TIMESTAMP
)
"""
SQLITE_DB = PATH + '/images.sqlite'
conn = None
def get_random_image():
query = "SELECT url FROM images"
cursor = conn.cursor()
cursor.execute(query)
result = cursor.fetchall()
return random.choice(result)[0]
def add_image(url, subreddit=None):
try:
if subreddit:
query = "INSERT INTO images (url, subreddit) VALUES (?, ?)"
conn.execute(query, (url, subreddit))
else:
query = "INSERT INTO images (url, subreddit) VALUES (?, NULL)"
conn.execute(query, (url,))
#
print termcolor.colored("# {url} added to DB".format(url=url), "cyan")
except sqlite3.IntegrityError:
print termcolor.colored("# the image {0} is already in the DB...".format(url), "red")
def get_all_images():
query = "SELECT url FROM images"
cursor = conn.cursor()
cursor.execute(query)
result = cursor.fetchall()
return result
def create_db():
"""
Create the DB if not exists.
"""
global conn
conn = sqlite3.connect(SQLITE_DB)
conn.executescript(SCHEMA)
def init(commit=True):
"""
Initialize the DB.
"""
global conn
if commit:
atexit.register(commit_and_close)
else:
atexit.register(close)
if not os.path.exists(SQLITE_DB):
create_db()
if not conn:
conn = sqlite3.connect(SQLITE_DB)
def commit():
"""
Commit.
"""
if conn:
conn.commit()
def close():
"""
Close.
"""
if conn:
conn.close()
def commit_and_close():
"""
Commit and close DB connection.
As I noticed, commit() must be called, otherwise changes
are not committed automatically when the program terminates.
"""
if conn:
conn.commit()
conn.close()
####################
if __name__ == "__main__":
init()
The module database.py is responsible for handling the database. When you need to work with the database, just include this file and call the appropriate function(s).
imgur album downloader
This little script can download all the images in an imgur gallery.
uploading an image to imgur
Problem
You want to upload an image to imgur.com anonymously, from your Python script of course.
Solution
#!/usr/bin/env python
"""
Upload an image to imgur anonymously.
# from jabbapylib.imgur import imgur
"""
from jabbapylib import config as cfg
import pycurl
import cStringIO
import untangle
def upload_from_computer(fpath):
"""
Upload an image from the local machine.
The return value is an XML string.
Beware! fpath must be normal string, not unicode!
With unicode it'll drop an error.
"""
response = cStringIO.StringIO()
c = pycurl.Curl()
values = [("key", cfg.IMGUR_KEY),
("image", (c.FORM_FILE, fpath))]
c.setopt(c.URL, "http://api.imgur.com/2/upload.xml")
c.setopt(c.HTTPPOST, values)
c.setopt(c.WRITEFUNCTION, response.write)
c.perform()
c.close()
return response.getvalue()
def upload_from_web(url):
"""
Upload an image from the web.
The return value is an XML string.
"""
response = cStringIO.StringIO()
c = pycurl.Curl()
values = [("key", cfg.IMGUR_KEY),
("image", url)]
c.setopt(c.URL, "http://api.imgur.com/2/upload.xml")
c.setopt(c.HTTPPOST, values)
c.setopt(c.WRITEFUNCTION, response.write)
c.perform()
c.close()
return response.getvalue()
def process(xml):
"""
Process the returned XML string.
"""
o = untangle.parse(xml)
url = o.upload.links.original.cdata
delete_page = o.upload.links.delete_page.cdata
print '# url: ', url
print '# delete page:', delete_page
##########################
## some simple wrappers ##
##########################
def upload_local_img(fpath):
"""
Upload a local image.
The return value is a tuple: (imgur_url, imgur_delete_url)
"""
xml = upload_from_computer(fpath)
o = untangle.parse(xml)
url = o.upload.links.original.cdata
delete_page = o.upload.links.delete_page.cdata
return (url, delete_page)
def upload_web_img(url):
"""
Upload a web image.
The return value is a tuple: (imgur_url, imgur_delete_url)
"""
xml = upload_from_web(url)
o = untangle.parse(xml)
url = o.upload.links.original.cdata
delete_page = o.upload.links.delete_page.cdata
return (url, delete_page)
#############################################################################
if __name__ == "__main__":
# img = '/tmp/test.jpg'
# xml = upload_from_computer(img)
# process(xml)
#
# url = 'http://...'
# xml = upload_from_web(url)
# process(xml)
#
# print upload_local_img('/tmp/test.jpg')
#
# print upload_web_img('http://...')
pass
You can find the latest version in my jabbapylib library.
Upload an image to imgur.com from Python
If you are familiar with reddit, you must have noticed that most images are hosted on imgur. I would like to upload several images from my computer and I want to collect their URLs on imgur. Let’s see how to do that.
Imgur has an API, this is what we’ll use. Anonymous upload is fine for my needs. For this you need to register and you get an API key. Under the examples there is a very simple Python code. When you execute it, pycurl prints the server’s XML response to the standard output. How to store that in a variable? From that XML we want to extract some data.
Here is an extended version of the uploader script:
#!/usr/bin/env python
import pycurl
import cStringIO
import untangle # XML parser
def upload_from_computer(image):
response = cStringIO.StringIO() # XML response is stored here
c = pycurl.Curl()
values = [
("key", your_api_key),
("image", (c.FORM_FILE, image))]
# OR: ("image", "http://example.com/example.jpg")]
# OR: ("image", "YOUR_BASE64_ENCODED_IMAGE_DATA")]
c.setopt(c.URL, "http://api.imgur.com/2/upload.xml")
c.setopt(c.HTTPPOST, values)
c.setopt(c.WRITEFUNCTION, response.write) # put the server's output in here
c.perform()
c.close()
return response.getvalue()
def process(xml):
o = untangle.parse(xml)
url = o.upload.links.original.cdata
delete_page = o.upload.links.delete_page.cdata
print 'url: ', url
print 'delete page:', delete_page
#############################################################################
if __name__ == "__main__":
img = '/tmp/something.jpg'
xml = upload_from_computer(img)
process(xml)
The tip for storing the XML output in a variable is from here. Untangle is a lightweight XML parser; more info here.
Python Adventures 