Skip to main content

Generate Python Library Files with Cookiecutter

Generate files sufficient for creation of a Python library using Cookiecutter and cookiecutter-pylibrary.

See this post for resources.

In [1]:
from pathlib import Path
import json
import shlex
from subprocess import check_output, Popen, PIPE, STDOUT
import os
In [2]:
# Define directory where cookiecutter templates live.
templates_path = Path(Path.home(), "projects", "cookiecutters")
In [3]:
pylibrary_path = templates_path.joinpath("cookiecutter-pylibrary")
In [4]:
# Get latest version from GitHub.
GIT_COMMAND_PREFIX = f"git -C {pylibrary_path} "
commands = (
    "reset --hard HEAD",
    "checkout master",
    "pull origin master",
)
GIT_HARD_RESET, = (f"{GIT_COMMAND_PREFIX}{command}" for command in commands[:1])

for command in commands:
    p = Popen(
        shlex.split(f"{GIT_COMMAND_PREFIX}{command}"),
        stdout=PIPE,
        stderr=STDOUT,
        universal_newlines=True,
    )
    print(p.stdout.read())
HEAD is now at ae3a882 Try to explain optional fields (issue #156).

Already on 'master'
Your branch is up to date with 'origin/master'.

From github.com:dm-wyncode/cookiecutter-pylibrary
 * branch            master     -> FETCH_HEAD
Already up to date.

In [5]:
cookiecutter_json_path = pylibrary_path.joinpath("cookiecutter.json")
paths = (templates_path, pylibrary_path, cookiecutter_json_path)

for path in paths:
    assert path.exists(), f"No dir {path}"
    
cookiecutter_context = json.loads(cookiecutter_json_path.read_text())
context_keys = tuple(cookiecutter_context.keys())

posts_dir = Path(os.path.abspath(os.curdir))

print(check_output(shlex.split(GIT_HARD_RESET)).decode())
HEAD is now at ae3a882 Try to explain optional fields (issue #156).

Truncate view of existing cookiecutter_context.

In [6]:
def head(item: dict, n=5):
    return {key: item[key] for key in list(item.keys())[:n]}
In [7]:
head(cookiecutter_context)
Out[7]:
{'full_name': 'Ionel Cristian Maries',
 'email': 'contact@ionelmc.ro',
 'website': 'https://blog.ionelmc.ro',
 'project_name': 'Nameless',
 'repo_name': "python-{{ cookiecutter.project_name|lower|replace(' ','-') }}"}

The code below was created with the intent to use pexpect with cookiecutter like it is done in this example.

--no-input option is a better way.

In [8]:
DEFAULT = None
NAME = "foo"
inputs = dict(
    default=DEFAULT,
    full_name="Don Morehouse",
    email="dm.wyncode@gmail.com",
    website="https://zip.apps.selfip.com/",
    project_name=NAME,
    repo_name=DEFAULT,
    repo_hosting=DEFAULT,
    repo_hosting_domain="apps.selfip.com",  # I host my own gitolite git server
    repo_username="gitolite3",  # TODO make git server URL into SSH version
    package_name=NAME,
    distribution_name=NAME,
    project_short_description=DEFAULT,
    release_date=DEFAULT,
    year_from=DEFAULT,
    year_to=DEFAULT,
    version=DEFAULT,
    license=DEFAULT,
    c_extension_support=DEFAULT,
    c_extension_optional=DEFAULT,
    c_extension_module=DEFAULT,
    c_extension_function=DEFAULT,
    c_extension_test_pypi=DEFAULT,
    c_extension_test_pypi_username=DEFAULT,
    test_matrix_configurator=DEFAULT,
    test_matrix_separate_coverage=DEFAULT,
    test_runner=DEFAULT,
    setup_py_uses_test_runner=DEFAULT,
    setup_py_uses_setuptools_scm=DEFAULT,
    pypi_badge=DEFAULT,
    pypi_disable_upload=DEFAULT,
    allow_tests_inside_package=DEFAULT,
    linter=DEFAULT,
    command_line_interface=DEFAULT,
    command_line_interface_bin_name=DEFAULT,
    coveralls=DEFAULT,
    coveralls_token=DEFAULT,
    codecov=DEFAULT,
    landscape=DEFAULT,
    scrutinizer=DEFAULT,
    codacy=DEFAULT,
    codacy_projectid=DEFAULT,
    codeclimate=DEFAULT,
    sphinx_docs=DEFAULT,
    sphinx_theme=DEFAULT,
    sphinx_doctest=DEFAULT,
    sphinx_docs_hosting=f"https://static.apps.selfip.com/{NAME}_docs/",
    travis=DEFAULT,
    travis_osx=DEFAULT,
    appveyor=DEFAULT,
    requiresio=DEFAULT,
)

Diff the inputs keys and context_keys

In [9]:
set(context_keys).difference(set(inputs.keys()))
Out[9]:
{'_extensions'}
In [10]:
EXTENSIONS = "_extensions"
inputs.update({EXTENSIONS: cookiecutter_context[EXTENSIONS]})
In [11]:
assert not set(context_keys).difference(
    set(inputs.keys())
), "inputs not current with cookiecutter_context"
In [12]:
custom_context = {
    key: inputs[key] or value for key, value in cookiecutter_context.items()
}
In [13]:
head(custom_context)
Out[13]:
{'full_name': 'Don Morehouse',
 'email': 'dm.wyncode@gmail.com',
 'website': 'https://zip.apps.selfip.com/',
 'project_name': 'foo',
 'repo_name': "python-{{ cookiecutter.project_name|lower|replace(' ','-') }}"}

Write the custom_context to the Cookiecutter template overwriting the existing one.

In [14]:
cookiecutter_json_path.write_text(json.dumps(custom_context))
Out[14]:
2444
In [15]:
import tempfile
from pprint import pprint

BASE_DIR = Path(Path.home(), "projects", "cookie-cut-py-lib")

NAME = "foo"
PROJECT_NAME = f"python-{NAME}"
PREFIX = f"{PROJECT_NAME}-"
WORKING_DIR_NAME = tempfile.mkdtemp(prefix=PREFIX, dir=BASE_DIR)
WORKING_DIR = BASE_DIR.joinpath(WORKING_DIR_NAME)
assert WORKING_DIR.exists(), "No working directory exists."
COMMAND = f"cookiecutter {pylibrary_path} --no-input -o {WORKING_DIR}"

pprint(check_output(shlex.split(COMMAND)).decode().splitlines(), indent=4)
[   'bootstrap create: '
    '/home/dmmmd/projects/cookie-cut-py-lib/python-foo-fygknkss/python-foo/.tox/bootstrap',
    'bootstrap installdeps: jinja2, matrix, tox',
    'bootstrap installed: '
    'configparser2==4.0.0,filelock==3.0.12,importlib-metadata==0.23,Jinja2==2.10.3,MarkupSafe==1.1.1,matrix==2.0.1,more-itertools==7.2.0,packaging==19.2,pluggy==0.13.0,py==1.8.0,pyparsing==2.4.2,six==1.12.0,toml==0.10.0,tox==3.14.0,virtualenv==16.7.7,zipp==0.6.0',
    "bootstrap run-test-pre: PYTHONHASHSEED='4072576528'",
    'bootstrap run-test: commands[0] | python ci/bootstrap.py --no-env',
    'Project path: '
    '/home/dmmmd/projects/cookie-cut-py-lib/python-foo-fygknkss/python-foo',
    'Wrote .travis.yml',
    'Wrote .appveyor.yml',
    'DONE.',
    '___________________________________ summary '
    '____________________________________',
    '  bootstrap: commands succeeded',
    '  congratulations :)',
    '',
    '################################################################################',
    '',
    '    Generating CI configuration ...',
    '',
    '',
    '################################################################################',
    '################################################################################',
    '',
    '    You have succesfully created `python-foo`.',
    '',
    '################################################################################',
    '',
    "    You've used these cookiecutter parameters:",
    '',
    "        _extensions:               ['jinja2_time.TimeExtension']",
    '        _template:                 '
    "'/home/dmmmd/projects/cookiecutters/cookiecutter-pylibrary'",
    "        allow_tests_inside_package: 'no'",
    "        appveyor:                  'yes'",
    "        c_extension_function:      'longest'",
    "        c_extension_module:        '_foo'",
    "        c_extension_optional:      'no'",
    "        c_extension_support:       'no'",
    "        c_extension_test_pypi:     'no'",
    "        c_extension_test_pypi_username: 'gitolite3'",
    "        codacy:                    'no'",
    "        codacy_projectid:          '[Get ID from "
    "https://app.codacy.com/app/gitolite3/python-foo/settings]'",
    "        codeclimate:               'no'",
    "        codecov:                   'yes'",
    "        command_line_interface:    'plain'",
    "        command_line_interface_bin_name: 'foo'",
    "        coveralls:                 'no'",
    "        coveralls_token:           '[Required for Appveyor, take it from "
    "https://coveralls.io/github/gitolite3/python-foo]'",
    "        distribution_name:         'foo'",
    "        email:                     'dm.wyncode@gmail.com'",
    "        full_name:                 'Don Morehouse'",
    "        landscape:                 'no'",
    "        license:                   'BSD 2-Clause License'",
    "        linter:                    'flake8'",
    "        package_name:              'foo'",
    "        project_name:              'foo'",
    "        project_short_description: 'An example package. Generated with "
    "cookiecutter-pylibrary.'",
    "        pypi_badge:                'yes'",
    "        pypi_disable_upload:       'no'",
    "        release_date:              'today'",
    "        repo_hosting:              'github.com'",
    "        repo_hosting_domain:       'apps.selfip.com'",
    "        repo_name:                 'python-foo'",
    "        repo_username:             'gitolite3'",
    "        requiresio:                'yes'",
    "        scrutinizer:               'no'",
    "        setup_py_uses_setuptools_scm: 'no'",
    "        setup_py_uses_test_runner: 'no'",
    "        sphinx_docs:               'yes'",
    '        sphinx_docs_hosting:       '
    "'https://static.apps.selfip.com/foo_docs/'",
    "        sphinx_doctest:            'no'",
    "        sphinx_theme:              'sphinx-rtd-theme'",
    "        test_matrix_configurator:  'no'",
    "        test_matrix_separate_coverage: 'no'",
    "        test_runner:               'pytest'",
    "        travis:                    'yes'",
    "        travis_osx:                'no'",
    "        version:                   '0.0.0'",
    "        website:                   'https://zip.apps.selfip.com/'",
    "        year_from:                 '2019'",
    "        year_to:                   '2019'",
    '',
    '    See .cookiecutterrc for instructions on regenerating the project.',
    '',
    '################################################################################',
    '',
    '    To get started run these:',
    '',
    '        cd python-foo',
    '        git init',
    '        git add --all',
    '        git commit -m "Add initial project skeleton."',
    '        git remote add origin '
    'git@apps.selfip.com:gitolite3/python-foo.git',
    '        git push -u origin master',
    '',
    '',
    '    To regenerate your .travis.yml or .appveyor.yml run:',
    '',
    '',
    '        tox -e bootstrap',
    '',
    '    You can also run:',
    '',
    '        ./ci/bootstrap.py',
    '',
    '']

The results look as expected.

In [16]:
os.chdir(WORKING_DIR)
print(Path(*Path(os.curdir).absolute().parts[3:]))
os.chdir(PROJECT_NAME)
projects/cookie-cut-py-lib/python-foo-fygknkss
In [17]:
!ls
AUTHORS.rst    CONTRIBUTING.rst  MANIFEST.in	 setup.cfg  tests
CHANGELOG.rst  docs		 pyproject.toml  setup.py   tox.ini
ci	       LICENSE		 README.rst	 src
In [18]:
p = Popen(shlex.split("tox -p auto"), stdout=PIPE, stderr=STDOUT)
p.wait()
output = p.stdout.read()
In [19]:
lines = output.decode().splitlines()
START = "  clean: commands succeeded"
pprint("\n".join(lines[lines.index(START) + 1:]))
('ERROR:   check: parallel child exit code 1\n'
 'ERROR:   docs: parallel child exit code 1\n'
 '  py27: commands succeeded\n'
 'ERROR:   py34: parallel child exit code 1\n'
 'ERROR:   py35: parallel child exit code 1\n'
 '  py36: commands succeeded\n'
 '  py37: commands succeeded\n'
 'ERROR:   pypy: parallel child exit code 1\n'
 'ERROR:   pypy3: parallel child exit code 1\n'
 '  report: commands succeeded')

Install the library

In [20]:
!pip install -e .
Obtaining file:///home/dmmmd/projects/cookie-cut-py-lib/python-foo-fygknkss/python-foo
  Installing build dependencies ... done
  Getting requirements to build wheel ... done
    Preparing wheel metadata ... done
Installing collected packages: foo
  Running setup.py develop for foo
Successfully installed foo

Run the foo executable

In [21]:
!foo
['/home/dmmmd/.virtualenvs/seven-notebooks/bin/foo']

Source for executable 'foo'

This code was autogenerated by the Cookiecutter template.

In [22]:
# %load src/foo/cli.py
"""
Module that contains the command line app.

Why does this file exist, and why not put this in __main__?

  You might be tempted to import things from __main__ later, but that will cause
  problems: the code will get executed twice:

  - When you run `python -mfoo` python will execute
    ``__main__.py`` as a script. That means there won't be any
    ``foo.__main__`` in ``sys.modules``.
  - When you import __main__ it will get executed again (as a module) because
    there's no ``foo.__main__`` in ``sys.modules``.

  Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration
"""
import sys


def main(argv=sys.argv):
    """
    Args:
        argv (list): List of arguments

    Returns:
        int: A return code

    Does stuff.
    """
    print(argv)
    return 0

Display coverage index.html generated during tox.

In [23]:
from IPython.display import display, HTML
In [24]:
index_file = Path('htmlcov', 'index.html')
display(HTML(index_file.read_text()))
Coverage report
Hide keyboard shortcuts

Hot-keys on this page

n s m x b p c   change column sorting

Module statements missing excluded branches partial coverage
Total 12 4 0 2 0 57.14%
src/foo/__init__.py 1 0 0 0 0 100.00%
src/foo/__main__.py 4 4 0 2 0 0.00%
src/foo/cli.py 4 0 0 0 0 100.00%
tests/test_foo.py 3 0 0 0 0 100.00%

No items found using the specified filter.

Display auto generated documentation.

This documentation was auto-generated from the library code when tox was run.

In [25]:
doc_index_file = Path('dist', 'docs', 'index.html')
display(HTML(doc_index_file.read_text()))
Contents — foo 0.0.0 documentation

Todo

  • Learn more about Tox
  • Learn more about the other options the template offers.
  • Tweak the template to use private Git server Gitolite

Notes to self.

  • tox -p auto will throw errors without having done a git commit first.
  • packaging step: python setup.py sdist

This step not found in the tox.ini generated by cookiecutter-pylibrary

Note that for this operation the same Python environment will be used as the one tox is installed into (therefore you need to make sure that it contains your build dependencies). Skip this step for application projects that don’t have a setup.py