Snippets Celery / Basics

Celery Basics

By Marcelo Fernandes Jul 12, 2017

Installing Celery


# pip is your friend.

pip install celery

Dependencies


# we will be using redis both as backend and message broker
sudo apt-get install redis

# run redis
redis-server

Sample Project Folder


project/
        __init__.py
        celery_config.py
        celery_fetch.py
        celery_run.py

Files

celery_config.py

from celery import Celery

app = Celery('celery_config',
             backend='redis://localhost',
             broker='redis://localhost:6379/0',
             include=['celery_fetch'])

app.conf.task_serializer = 'json'
app.conf.result_serializer = 'json'


celery_fetch.py
from celery_config import app
import requests
from celery import group


# --- Config Using Celery --- #

@app.task
def fetch_url(url):
    return requests.get(url).status_code, url


def func(urls):
    # make calls in parallel and retrieve them in order.
    return group(fetch_url.s(url) for url in urls)().get()


# --- Config Not Using Celery --- #

def fetch_url2(url):
    return requests.get(url).status_code, url


def func2(urls):
    result = []
    for url in urls:
        result.append(fetch_url2(url))

    return result


celery_run.py

from celery_fetch import func, func2
import time

urls = ["http://google.com",
        "https://amazon.in",
        "https://facebook.com",
        "https://twitter.com",
        "https://alexa.com"]

# ------ Using Celery ------- #

start = time.time()
results = func(urls)


print(results)
# [[200, 'http://google.com'], [503, 'https://amazon.in'], [200, 'https://facebook.com'], [200, 'https://twitter.com'], [200, 'https://alexa.com']]

print("It took", time.time() - start, "seconds using celery")
# It took 2.8189401626586914 seconds using celery

# ----- NOT using Celery -------- #
start_2 = time.time()
results2 = func2(urls)

print(results2)
# [(200, 'http://google.com'), (503, 'https://amazon.in'), (200, 'https://facebook.com'), (200, 'https://twitter.com'), (200, 'https://alexa.com')]

print("It took", time.time() - start_2, "seconds NOT using celery")
# It took 7.893500089645386 seconds NOT using celery


Running


# inside the project folder

celery worker -A celery_config -l info -c 5


in another terminal, in the project folder

python3 celery_run.py


Notes