Category: python

[Python] – XML Validator against XML Schema (XSD)

Hi,

This is the code:

from easyxsd import *
import sys

# Load XML Schema (.xsd file)
xsd = xsd_from_file(sys.argv[1])

# Load XML File
xml = xml_from_file(sys.argv[2])

# Validate
list_errors = validate_with_errors(xml, xsd)

# Open output file (validation response)
f = open(sys.argv[3],'w')

for error in list_errors:
    f.write(str(error) + '\n')

f.close()

mime_exeDownload here an executable version for Windows

To execute:
validate_xml.exe C:\Documents\MyFolder\schema.xsd C:\Documents\MyFolder\myfile.xml C:\Documents\MyFolder\outputvalidate.txt

The shortcut works too:
validate_xml.exe schema.xsd myfile.xml outputvalidate.txt

Credits:
https://github.com/gnrfan/python-easyxsd

Backup File Utility in Python [Tkinter application]

Hi readers,

This is a graphic software (tkinter) in Python to back up a file on your computer. A backup folder is created in the current directory of the selected file and then every x minutes a backup of the file is done, respecting the maximum number of files in the backup directory.

backup_app

 

backup_app_dir

 

import hashlib, time, shutil, os, threading
from datetime import datetime
from Tkinter import Label, Spinbox, Listbox, Scrollbar, Tk, E, W, S, N, Button, Frame
import tkFileDialog
from os.path import expanduser
 
class App(Frame):
 
    def __init__(self, parent):
 
        Frame.__init__(self, parent)
        self.parent = parent
 
        # Class Variables
        self.directory_selected = ''
        self.filename_selected = ''
        self.backup_folder_name = 'backup'
        self.num_max_files = None
        self.time_for_backup = None
        self.full_file_path = ''
        self.backup_running = False
        self.hash = None
 
        self.initUI()
 
    def initUI(self):
        self.parent.title("Application to Backup a File")
 
        self.label_minutes = Label( text='Time for Backup (Minutes): ', height=2)
        self.spinbox_minutes = Spinbox( from_=1, to=100, width=5)
        self.spinbox_minutes.delete(0, 'end')
        self.spinbox_minutes.insert(0,5)
 
        self.label_numfiles = Label( text='Maximum number files in backup directory: ')
        self.spinbox_numfiles = Spinbox( from_=1, to=100, width=5)
        self.spinbox_numfiles.delete(0, 'end')
        self.spinbox_numfiles.insert(0,10)
 
        self.scrollbar_loglist = Scrollbar()
        self.listbox_loglist = Listbox(yscrollcommand=self.scrollbar_loglist.set, width=70)
        self.scrollbar_loglist.config(command=self.listbox_loglist.yview)
 
        self.button_file_choose = Button(text='Choose a file for backup ...', command=self.click_button_start_backup)
        self.button_stop_backup = Button(text='Stop Backup', command=self.click_button_stop_backup)
        self.button_stop_backup['state'] = 'disabled'
        self.button_exit = Button(text='Exit', command=self.click_button_exit)
 
        # Here the UI components are designed using grid layout
        self.label_minutes.grid(row=0, sticky=E)
        self.spinbox_minutes.grid(row=0, column=1, sticky=W)
        self.label_numfiles.grid(row=1, sticky=E)
        self.spinbox_numfiles.grid(row=1, column=1, sticky=W)
        self.listbox_loglist.grid(row=2, column=0, sticky=W+E+N+S, columnspan=3)
        self.scrollbar_loglist.grid(row=2, column=3, sticky=N+S)
        self.button_file_choose.grid(row=3, column=0)
        self.button_stop_backup.grid(row=3, column=1)
        self.button_exit.grid(row=3, column=2)
 
        # define options for opening or saving a file
        self.file_opt = options = {}
        options['initialdir'] = expanduser("~")
        options['parent'] = self
        options['title'] = 'Select a file for backup ...'
 
    def get_ask_open_file(self):
        return tkFileDialog.askopenfilename(**self.file_opt)
 
    def get_current_datetime(self):
        return datetime.strftime(datetime.now(), "%d%m%Y_%H%M%S")
 
    def get_current_datetime_formatted(self):
        return datetime.strftime(datetime.now(), "[%d/%m/%Y %H:%M:%S] - ")
 
    def log_action(self, msg):
        self.listbox_loglist.insert(0, self.get_current_datetime_formatted() + msg)
 
    def do_file_backup(self):
        curr_datetime = self.get_current_datetime()
        nom_arquivo = self.filename_selected.split('.')[0] + '_' + curr_datetime + '.' + self.filename_selected.split('.')[1]
        shutil.copyfile(self.full_file_path, self.directory_selected + os.sep + self.backup_folder_name + os.sep + nom_arquivo)
        self.log_action('Backup Done - ' + nom_arquivo)
 
    def get_file_hash_md5(self, file):
       md5 = hashlib.md5()
       with open(file, "rb") as f:
           for block in iter(lambda: f.read(128), ""):
               md5.update(block)
       return md5.hexdigest()
 
    def listdir_fullpath(self, d):
        return [os.path.join(d, f) for f in os.listdir(d)]
 
    def delete_oldest_files(self):
        num_total_files_in_directory = len(self.listdir_fullpath(self.directory_selected + os.sep + self.backup_folder_name))
        num_files_to_be_removed = num_total_files_in_directory - self.num_max_files
        if num_files_to_be_removed > 0:
            self.log_action('Removing ' + str(num_files_to_be_removed) + ' old files')
        for i in range(num_files_to_be_removed):
            file_to_remove = self.get_oldest_file_from_directory(self.directory_selected + os.sep + self.backup_folder_name)
            os.remove(file_to_remove)
            self.log_action('An old file was removed - ' + file_to_remove.split(os.sep)[-1])
 
    def get_oldest_file_from_directory(self, dir):
        return min(self.listdir_fullpath(dir), key=os.path.getctime)
 
    def start_loop_backup_job(self):
        if not self.backup_running:
            return
        self.delete_oldest_files()
        if self.hash != self.get_file_hash_md5(self.full_file_path):
            self.do_file_backup()
        else:
            self.log_action('The file was not changed since last check')
        self.hash = self.get_file_hash_md5(self.full_file_path)
        self.parent.after(self.time_for_backup * 1000 * 60, self.start_loop_backup_job)
 
    def click_button_start_backup(self):
        file_choosed = self.get_ask_open_file()
        if not os.path.isfile(file_choosed):
            return
        self.log_action('Backup has been started')
        self.backup_running = True
        self.button_file_choose['state'] = 'disabled'
        self.button_stop_backup['state'] = 'normal'
        self.directory_selected = os.path.split(os.path.abspath(file_choosed))[0]
        self.filename_selected = os.path.split(os.path.abspath(file_choosed))[1]
        self.full_file_path = self.directory_selected + os.sep + self.filename_selected

        if not os.path.exists(self.directory_selected + os.sep + self.backup_folder_name):
            os.mkdir(self.directory_selected + os.sep + self.backup_folder_name)

        self.time_for_backup = int(self.spinbox_minutes.get())
        self.num_max_files = int(self.spinbox_numfiles.get())
 
        self.backup_running = True
        self.start_loop_backup_job()
 
    def click_button_stop_backup(self):
        self.button_file_choose['state'] = 'normal'
        self.button_stop_backup['state'] = 'disabled'
        self.log_action('Backup has been stopped')
        self.backup_running = False
 
    def click_button_exit(self):
        self.parent.destroy()
 
 
def main():
    root = Tk()
    root.eval('tk::PlaceWindow %s center' % root.winfo_pathname(root.winfo_id()))
    app = App(root)
    root.mainloop()
 
if __name__ == '__main__':
    main()

Python – Unique Collection Class – HashCode and Equals – (Like Java Set)

Hi readers,

I will demonstrate in this post as we do in Python to have a collection of single objects (Set), implementing the standard methods key, hashcode and equals, so that the objects are not repeated.

class MyCustomClass:
    def __init__(self, at1, at2, at3):
        self.at1 = at1
        self.at2 = at2
        self.at3 = at3
    def __key(self):
        return (self.at2)
    def __eq__(x, y):
        return x.__key() == y.__key()
    def __hash__(self):
        return hash(self.__key())

my_object_1 = MyCustomClass('1', '2', '3')
my_object_2 = MyCustomClass('4', '5', '6')
my_object_3 = MyCustomClass('7', '8', '9')
my_object_2_dup = MyCustomClass('4', '5', '6')

set_collection = set()

set_collection.add(my_object_1)
set_collection.add(my_object_2)
set_collection.add(my_object_3)
set_collection.add(my_object_2_dup)

for object in set_collection:
    print object.at2

The output of this script is:
8
2
5

Note that i added 4 objects, but only three is added because the second attribute is the Key.

See You,
Victor Jabur

Compiling MySQLdb 1.2.3 on Windows 32 and 64 – Without need the MySQL Database 5.5 installed

Hi,

If you wish the module already compiled, access here: http://blog.victorjabur.com/2011/06/08/modules-python-library-compiled-for-windows-32-and-64-unofficial-windows-binaries-for-python/

First Thing: Follow this main steps to prepare the pre-requirements for compile this module:

http://blog.victorjabur.com/2011/06/05/compiling-python-2-7-modules-on-windows-32-and-64-using-msvc-2008-express/

1 – Compiling MySQLdb 1.2.3 for Mysql 5.5 database

1.1 – You will need to download the latest version of MySQL Connector C here: http://dev.mysql.com/downloads/connector/c/

* Important: Make the download properly for your platform (32 or 64 bits).

1.2 – Make the download of MySQLdb plugin here and extract to any folder: http://sourceforge.net/projects/mysql-python/

1.3 – Edit this file (MySQL-python-1.2.3/setup_windows.py) inside of folder extracted:

At Line 7 of the file, comment the two follow lines and add another, like this:

    #serverKey = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, options['registry_key'])
    #mysql_root, dummy = _winreg.QueryValueEx(serverKey,'Location')
    mysql_root = ''

At Line 19, put the path to your Connector C installed:

    library_dirs = [ os.path.join(mysql_root, r'lib\opt'), 'C:\Program Files\MySQL\MySQL Connector C 6.0.2\lib\opt' ]
    libraries = [ 'kernel32', 'advapi32', 'wsock32', client ]
    include_dirs = [ os.path.join(mysql_root, r'include'), 'C:\Program Files\MySQL\MySQL Connector C 6.0.2\include' ]
    extra_compile_args = [ '/Zl' ]

Important: Specify the correct path for your platform (32 or 64 bits). Above the platform is 64 bits, to 32 bits use this path: “C:\Program Files (x86)\MySQL\MySQL Connector C 6.0.2”.

1.4 – Make the build:

python setup.py build --compiler msvc
python setup.py bdist_wininst

It’s finish.

The advantage os this technique an a comparison with others posts is:

1 – The MySQL Database is not required on machine (only the connector c)
2 – Not is necessary make the update on the windows registry:

# The Windows registry key for MySQL.
# This has to be set for Windows builds to work.
# Only change this if you have a different version.
registry_key = SOFTWARE\MySQL AB\MySQL Server 5.0

Credits and References to this post:

http://chaos.weblogs.us/archives/327
http://www.fuyun.org/2009/12/install-mysql-for-python-on-windows/

Bye,
Victor Jabur

Modules Python Library Compiled for Windows 32 and 64 – Unofficial Windows Binaries for Python

Unofficial Windows Binaries for Python Extension Packages

This is a list of python modules compiled for me, the intention is to create a complete list of windows modules for python, this post is always updated to extend this list. I will need your help with suggestions about wich modules will be available here and links about already existents modules on the web. The files are provided “as is” without warranty or support of any kind. The entire risk as to the quality and performance is with you. The files are unofficial (meaning: informal, unrecognized, personal, unsupported) and made available for testing and evaluation purposes only.

Do you need another module that isn’t available at this list ? Send an e-mail and request your module.
victorjabur@gmail.com

Links with more plugins on Web:

http://www.lfd.uci.edu/~gohlke/pythonlibs/

—————————————————————————————————————–
Django – Framework Webhttps://www.djangoproject.com/
Django is a high-level Python Web framework that encourages rapid development and clean, pragmatic design.

Django-1.3.1.win32-py2.7.exe [6.4 Mb] [Python 2.7] [32 bit] [JUN 08, 2011]
Django-1.3.1.win-amd64-py2.7.exe [6.5 Mb] [Python 2.7] [64 bit] [JUN 08, 2011]

—————————————————————————————————————–
mpi4py – Plugin MPI for Pythonhttp://mpi4py.scipy.org/http://en.wikipedia.org/wiki/Message_Passing_Interface
Message Passing Interface (MPI) is an API specification that allows processes to communicate with one another by sending and receiving messages. Besides many other applications, it is a de facto standard for parallel programs running on computer clusters and supercomputers, where the cost of accessing non-local memory is high. MPI was created since 1992 by William Gropp, Ewing Lusk and others, a first standard appeared in 1994.

mpi4py-1.2.2.win32-py2.7.exe [445 Kb] [Python 2.7] [32 bit] [JUN 08, 2011]
mpi4py-1.2.2.win-amd64-py2.7.exe [522 Kb] [Python 2.7] [64 bit] [JUN 08, 2011]

—————————————————————————————————————–
MySQLdb – Plugin Connector of Database MySQL to Pythonhttp://sourceforge.net/projects/mysql-python/
Message Passing Interface (MPI) is an API specification that allows processes to communicate with one another by sending and receiving messages. Besides many other applications, it is a de facto standard for parallel programs running on computer clusters and supercomputers, where the cost of accessing non-local memory is high. MPI was created since 1992 by William Gropp, Ewing Lusk and others, a first standard appeared in 1994.

MySQL-python-1.2.3.win32-py2.7.exe [1 Mb] [Python 2.7] [32 bit] [JUN 08, 2011]
MySQL-python-1.2.3.win-amd64-py2.7.exe [1.1 Mb] [Python 2.7] [64 bit] [JUN 08, 2011]

—————————————————————————————————————–
Paramiko – SSH2 for Pythonhttp://www.lag.net/paramiko/
Paramiko is a module for python 2.2 (or higher) that implements the SSH2 protocol for secure (encrypted and authenticated) connections to remote machines. unlike SSL (aka TLS), SSH2 protocol does not require heirarchical certificates signed by a powerful central authority. you may know SSH2 as the protocol that replaced telnet and rsh for secure access to remote shells, but the protocol also includes the ability to open arbitrary channels to remote services across the encrypted tunnel — this is how sftp works, for example. It is written entirely in python (no C or platform-dependent code) and is released under the GNU LGPL (lesser GPL).

paramiko-1.7.7.1.win32-py2.7.exe [331 Kb] [Python 2.7] [32 bit] [JUN 08, 2011]
paramiko-1.7.7.1.win-amd64-py2.7.exe [356 Kb] [Python 2.7] [64 bit] [JUN 08, 2011]

—————————————————————————————————————–
PyAMF – AMF for Pythonhttp://www.pyamf.org
PyAMF provides Action Message Format (AMF) support for Python that is compatible with the Adobe Flash Player. It includes integration with Python web frameworks like Django, Pylons, Twisted, SQLAlchemy, web2py and more.

PyAMF-0.6.1.win32-py2.7.exe [508 Kb] [Python 2.7] [32 bit] [JUN 08, 2011]
PyAMF-0.6.1.win-amd64-py2.7.exe [551 Kb] [Python 2.7] [64 bit] [JUN 08, 2011]

—————————————————————————————————————–
PyCrypto – Python Cryptography Toolkithttps://www.dlitz.net/software/pycrypto/http://sourceforge.net/projects/pycrypto/http://www.amk.ca/python/code/crypto.html
This is a collection of cryptographic algorithms and protocols, implemented for use from Python.

pycrypto-2.3.win32-py2.7.exe [545 Kb] [Python 2.7] [32 bit] [JUN 08, 2011]
pycrypto-2.3.win-amd64-py2.7.exe [572 Kb] [Python 2.7] [64 bit] [JUN 08, 2011]

—————————————————————————————————————–
PyXMLSec – PyXMLSec is a set of Python bindings for the XML Security Library.http://pyxmlsec.labs.libre-entreprise.org/

pyxmlsec-0.3.0.win32-py2.7.exe [255 Kb] [Python 2.7] [32 bit] [SET 29, 2012]
libxml2-python-2.7.7.win32-py2.7.exe (Pre-Requisite) [1.5Mb] [Python 2.7] [32 bit] [SET 29, 2012]
dll_pyxmlsec_0.3.0_py2.7.rar (Pre-Requisite Dlls - Put this on your C:\Python27) [998Kb] [Python 2.7] [32 bit] [SET 29, 2012]

—————————————————————————————————————–
PyBluez- PyBluez is an effort to create python wrappers around system Bluetooth resources to allow Python developers to easily and quickly create Bluetooth applications.http://code.google.com/p/pybluez/

PyBluez-0.18.win32-py2.7.exe [286 Kb] [Python 2.7] [32 bit] [SET 29, 2012]

—————————————————————————————————————–
DbfPy – DbfPy is a python-only module for reading and writing DBF-files. It was created by Jeff Kunce and then modified by Hans Fiby.http://dbfpy.sourceforge.net/

dbfpy-2.2.5_python_27_.win32.exe [211 Kb] [Python 2.7] [32 bit] [MAY 28, 2013]
dbfpy-2.2.5.win-python_27_amd64.exe [238 Kb] [Python 2.7] [64 bit] [MAY 28, 2013]

—————————————————————————————————————–
PyXML – PyXml is a python module to work with the Xml format. It was created on 2000 year. PyXML is no longer maintained.http://sourceforge.net/projects/pyxml/

PyXML-0.8.4.win32-py2.7.exe [1 Mb] [Python 2.7] [32 bit] [AUG 19, 2014]
PyXML-0.8.4.win-amd64-py2.7.exe [1 Mb] [Python 2.7] [64 bit] [AUG 19, 2014]

—————————————————————————————————————–

Do you need another module that isn’t available at this list ? Send an e-mail and request your module.
victorjabur@gmail.com

Compiling Python 2.7 Modules on Windows 32 and 64 using MSVC++ 2008 Express

Hello,

On this post i will explain how to build, compile, install and distribute python modules on Windows using Microsoft Visual C++ Express Edition.
This post will be constantly updated to cover future updates of python, windows and msvc++ versions.

For this example i will use the PyCrypto – http://pycrypto.org because this is an example that don’t have packages for windows x64 on the web.

Observation: Don’t use Microsoft Visual C++ Express Edition 2010 to build python modules, because this will not work due to Python 2.7 was built using the 2008 version. This is an error that occurs when you try to build PyCrypto and Paramiko using the 2010 version and execute the import module:

>>> import paramiko
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "C:\Python27\lib\site-packages\paramiko\__init__.py", line 69, in <module>
    from transport import SecurityOptions, Transport
  File "C:\Python27\lib\site-packages\paramiko\transport.py", line 32, in <module>
    from paramiko import util
  File "C:\Python27\lib\site-packages\paramiko\util.py", line 32, in <module>
    from paramiko.common import *
  File "C:\Python27\lib\site-packages\paramiko\common.py", line 98, in <module>
    from Crypto import Random
  File "C:\Python27\lib\site-packages\Crypto\Random\__init__.py", line 28, in <module>
    import OSRNG
  File "C:\Python27\lib\site-packages\Crypto\Random\OSRNG\__init__.py", line 34, in <module>
    from Crypto.Random.OSRNG.nt import new
  File "C:\Python27\lib\site-packages\Crypto\Random\OSRNG\nt.py", line 28, in <module>
    import winrandom
ImportError: DLL load failed: The specified module could not be found.

1 – Building and Installing PyCrypto Module for Windows 7 64 bits:

1.1 – You must have installed the Python 64 bits version: http://www.python.org/ftp/python/2.7.1/python-2.7.1.amd64.msi

1.2 – You should install the C Compiler for Windows – Microsoft Visual C++ Express Edition 2008: available here:http://www.microsoft.com/express/Downloads/#Visual_Studio_2008_Express_Downloads

ISO File to Download: http://www.microsoft.com/express/Downloads/#2008-All

1.3 – You should install the Microsoft Windows SDK for Windows 7 and .NET Framework 3.5 SP1: available here: http://www.microsoft.com/downloads/en/details.aspx?FamilyID=c17ba869-9671-4330-a63e-1fd44e0e2505

ISO File To Download (64 bits): http://download.microsoft.com/download/2/E/9/2E911956-F90F-4BFB-8231-E292A7B6F287/GRMSDKX_EN_DVD.iso

This is required because the Express Edition 2008 C++ don’t contains the 64 bits compiler. This is required only for Windows 7 64 bits version.

Important: Don’t use the “Microsoft Windows SDK for Windows 7 and .NET Framework 4” because it’s not compatible with msvc++ express 2008 edition.

1.4 – Install the Python Setup Tools available here: http://pypi.python.org/pypi/setuptools#downloads

1.5 – Include in your Advanced Variables Environment the binaries of Python. Right click at “My Computer” icon -> Properties -> Advanced Environment and edit your Path Variable including this two directories there:
Path = C:\Python27\Scripts;C:\Python27; + Path

1) C:\Python27\Scripts
2) C:\Python27

1.6 – Copy this file:

C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat

To this follow folder and rename the file (vcvars64.bat to vcvarsamd64.bat):

C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\amd64\vcvarsamd64.bat

1.7 – Edit the file msvc9compiler.py inside of directory C:\Python27\Lib\distutils\msvc9compiler.py

After the line 651 approximately, find this line: – ld_args.append(‘/MANIFESTFILE:’ + temp_manifest)
Add the following line after the above line:

ld_args.append('/MANIFEST')

1.8 – Edit the file msvccompiler.py inside of directory C:\Python27\Lib\distutils\msvccompiler.py

At line 153 approximately, insert this line: return 9.0, as following, in this piece of code:

def get_build_version():
    """Return the version of MSVC that was used to build Python.

    For Python 2.3 and up, the version number is included in
    sys.version.  For earlier versions, assume the compiler is MSVC 6.
    """
    return 9.0
    prefix = "MSC v."
    i = string.find(sys.version, prefix)
    if i == -1:
        return 6

1.9 – Certify that exists the follow environment variable in your system, if don’t exist create a new one:

Name: VS90COMNTOOLS
Value: C:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools\

1.10 – Extract the file pycrypto-2.3.tar.gz that you downloaded before, open a dos command and access the pycrypto-2.3 folder

1.11 – Execute this command to compile, build and install the module:

python setup.py build --compiler msvc
python setup.py install

1.12 – Test if the package was generated successfully executing an import Crypto at Python shell.

1.13 – It’s finish. Optionally, you can distribute your compiled modules as an windows executable (.exe) file, executing this simple command on folder of your module, this case is pycrypto-2.3:

python setup.py bdist_wininst

As a result you will get a executable file with graphical interface created inside of folder “dist” and other people can install this executable file without to prepare a complex environment for build your own module. This is my result file of this process. http://arquivos.victorjabur.com/python/modules/pycrypto-2.3.win-amd64-py2.7.exe

2 – Building and Installing PyCrypto Module for Windows All Versions 32 bits:

2.1 – You must have installed the Python 32 bits version: http://www.python.org/ftp/python/2.7.1/python-2.7.1.msi

2.2 – You should install the C Compiler for Windows – Microsoft Visual C++ Express Edition 2008: available here:http://www.microsoft.com/express/Downloads/#Visual_Studio_2008_Express_Downloads

ISO File to Download: http://www.microsoft.com/express/Downloads/#2008-All

2.3 – Install the Python Setup Tools available here: http://pypi.python.org/pypi/setuptools#downloads

2.4 – Include in your Advanced Variables Environment the binaries of Python. Right click at “My Computer” icon -> Properties -> Advanced Environment and edit your Path Variable including this two directories there:
Path = C:\Python27\Scripts;C:\Python27; + Path

1) C:\Python27\Scripts
2) C:\Python27

2.5 – Edit the file msvc9compiler.py inside of directory C:\Python27\Lib\distutils\msvc9compiler.py

After the line 651 approximately, find this line: – ld_args.append(‘/MANIFESTFILE:’ + temp_manifest)
Add the following line after the above line:

ld_args.append('/MANIFEST')

2.6 – Edit the file msvccompiler.py inside of directory C:\Python27\Lib\distutils\msvccompiler.py

At line 153 approximately, insert this line: return 9.0, as following, in this piece of code:

def get_build_version():
    """Return the version of MSVC that was used to build Python.

    For Python 2.3 and up, the version number is included in
    sys.version.  For earlier versions, assume the compiler is MSVC 6.
    """
    return 9.0
    prefix = "MSC v."
    i = string.find(sys.version, prefix)
    if i == -1:
        return 6

2.7 – Certify that exists the follow environment variable in your system, if don’t exist create a new one:

Name: VS90COMNTOOLS
Value: C:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools\

2.8 – Extract the file pycrypto-2.3.tar.gz that you downloaded before, open a dos command and access the pycrypto-2.3 folder

2.9 – Execute this command to compile, build and install the module:

python setup.py build --compiler msvc
python setup.py install

2.10 – Test if the package was generated successfully executing an import Crypto at Python shell.

2.11 – It’s finish. Optionally, you can distribute your compiled modules as an windows executable (.exe) file, executing this simple command on folder of your module, this case is pycrypto-2.3:

python setup.py bdist_wininst

As a result you will get a executable file with graphical interface created inside of folder “dist” and other people can install this executable file without to prepare a complex environment for build your own module. This is my result file of this process. http://arquivos.victorjabur.com/python/modules/pycrypto-2.3.win32-py2.7.exe

Credits and References to this post:

http://nukeit.org/compile-python-2-7-packages-with-visual-studio-2010-express/
http://yorickdowne.wordpress.com/2010/12/22/compiling-pycrypto-on-win7-64/
http://www.lfd.uci.edu/~gohlke/pythonlibs/
http://www.fuyun.org/2009/12/install-mysql-for-python-on-windows/
http://mattptr.net/2010/07/28/building-python-extensions-in-a-modern-windows-environment/

Good Bye.
Victor Jabur

Script Python para Deploy Automatizado usando MD5 e SFTP

Boa Noite pessoal,

Contratei uma hospedagem na Locaweb recentemente para meus sites em Django / Python e eu como sempre querendo automatizar tudo o tempo todo, criei um script em Python que realiza o deploy da minha aplicação lá no servidor remoto.

Objetivo: Copiar uma pasta e todo o seu conteúdo recursivamente para o servidor usando Secure File Transfer Protocol (SFTP), sincronizando as duas pastas (local e remota), isto é, identificar quais arquivos modificaram (usando md5) e copiar apenas eles para o servidor, removendo todos os arquivos e pastas remotas que foram deletados localmente desde a última sincronização. Tudo isto de um jeito super rápido, ocupando o mínimo de banda possível e feito com apenas 1 clique. Semelhante ao deploy do Google App Engine, para quem já utilizou.

Características:

  1. Biblioteca Paramiko – Estou utilizando uma biblioteca poderosa que implementa a parte de comunicação remota (SSH e SFTP e muito mais), chamada Paramiko.
  2. Índice MD5 – Para comparar arquivos locais e remotos, utilizo MD5, isto é, uma espécie de checksum / hashcode, que lê os bytes de um arquivo e me gera uma string única, sobre a qual eu relizo a comparação. Para não ter que fazer download do arquivo remoto apenas para calcular o md5, criei um índice, que nada mais é do que um dicionário em python que contém como chave o caminho completo do arquivo remoto e como valor o seu md5. Atualizo esse índice durante a sincronização.
  3. Arquivo de Configuração (Properties) – Estou utilizando um arquivo properties para isolar os dados de conexão ao servidor, tais como usuário e senha e mantê-los em uma pasta em separado.
  4. Barra de Progresso – Tentei implementar uma espécie de barra de progresso em modo texto, que imprime 100 caracteres “ponto”, e cada caractere representa 1% do upload do arquivo durante a cópia para o servidor.

Não se esqueçam de instalar a biblioteca paramiko, tem informações aqui no site do autor: http://www.lag.net/paramiko/

Aqui estão os pacotes necessários para as pessoas que utilizam o windows:

– Windows 64 bits (Python 2.7)

paramiko-1.7.7.1.win-amd64-py2.7.exe
pycrypto-2.3.win-amd64-py2.7.exe

– Windows 32 bits (Python 2.7)

paramiko-1.7.7.1.win32-py2.7.exe
pycrypto-2.3.win32-py2.7.exe

Para instalar em linux basta fazer o download do .tar.gz, acessar o diretório expandido e digitar o comando easy_install .

Vou postar agora o código fonte, são 2 arquivos, um para a configuração (properties – .ini) e o outro realmente que realiza o deploy (.py).

Arquivo de Configuração: settings.ini

[deploy]
HOSTNAME: meuservidor.com ou 187.200.123.12
PORT: 22
USERNAME: usuario
PASSWORD: senha
DIR_LOCAL_RAIZ: /app/pythonProjects
DIR_REMOTO_RAIZ: /home/storage/abc/julianajabur
WSGI: /home/storage/abc/julianajabur/public_html/julianajabur/index.wsgi
INDICE_MD5: /home/storage/abc/julianajabur/python_conf/julianajabur/indexmd5

Explicando os parâmetros acima:

  1. HOSTNAME = Endereço do servidor, pode ser um ip ou um nome DNS
  2. PORT = Porta de comunicação, 22 é a porta padrão para ftp
  3. USERNAME = usuário
  4. PASSWORD = senha
  5. DIR_LOCAL_RAIZ = caminho raiz da pasta que eu quero copiar, importante: aqui NÃO É a pasta a ser copiada, e sim apenas o caminho inicial dela, utilizo isso para fazer o de/para da estrutura de arquivos local versus a remota.
  6. DIR_REMOTO_RAIZ = idem ao de cima, só que aplicado à pasta remota.
  7. WSGI – aqui é o caminho até o seu arquivo WSGI, pois eu realizo uma cópia dele por último, para ter o mesmo efeito de touch arquivo.wsgi, indicando ao apache que sua aplicação foi mudada e deve ser atualizada.
  8. INDICE_WSGI = Caminho remoto onde vai ser gravado o arquivo de índice md5, observação, deve ser um diretório fora da pasta do seu projeto, para não atrapalhar no processo de sincronização de pastas.

Arquivo SFTPLocaweb.py

#! /usr/bin/python
# -*- coding: iso-8859-1 -*-

import os, paramiko, hashlib, sys, pickle
from stat import S_ISREG, S_ISDIR
from ConfigParser import RawConfigParser

class SFTPLocaweb:

    def __init__(self):
        self.caminhoArquivoConfiguracao = '../../../../python_conf/julianajabur/settings.ini'
        config = self.getConfigurationFile()
        self.hostname = config.get('deploy', 'HOSTNAME')
        self.port = config.getint('deploy', 'PORT')
        self.username = config.get('deploy', 'USERNAME')
        self.password = config.get('deploy', 'PASSWORD')
        self.dirLocalRaiz = config.get('deploy', 'DIR_LOCAL_RAIZ')
        self.dirRemotoRaiz = config.get('deploy', 'DIR_REMOTO_RAIZ')
        self.wsgi = config.get('deploy', 'WSGI')
        self.indice_md5 = config.get('deploy', 'INDICE_MD5')
        self.pastaOrigem = ''
        self.pastaDestino = ''
        self.sftp = None
        self.dicionario_md5 = {}
        self.statusTransferencia = 0    
        self.totais = {}    
        self.transport = None
  
    def getConfigurationFile(self):
        BASE_DIR = os.path.dirname(os.path.abspath(__file__))
        PYTHON_CONF = os.path.abspath(self.pathJoin(BASE_DIR, self.caminhoArquivoConfiguracao))
        config = RawConfigParser()
        config.read(PYTHON_CONF)
        return config        
    
    def getConexaoSSH(self):
        try:
            print 'Estabelecendo conexão com: ', self.hostname, self.port, '...'
            self.transport = paramiko.Transport((self.hostname, self.port))
            self.transport.connect(username=self.username, password=self.password, hostkey=None)
            self.sftp = paramiko.SFTPClient.from_transport(self.transport)
        except Exception, e:
            print '*** Erro ao se conectar com o servidor: %s: %s' % (e.__class__, e)
            sys.exit()
            try:
                self.transport.close()
            except:
                pass
  
    def pathJoin(self, raiz, diretorio):
        return os.path.join(raiz, diretorio).replace('\\', '/')
    
    def criarDiretorioRemoto(self, diretorio):
        try:
            self.sftp.mkdir(diretorio)
            print '    (diretório criado) ', diretorio
            return 1
        except IOError:
            print '    (diretório já existe) ', diretorio
            return 0
  
    def getDiretorioRemotoFromLocal(self, diretorioLocal):
        return diretorioLocal.replace(self.dirLocalRaiz, self.dirRemotoRaiz)

    def getDiretorioLocalFromRemoto(self, diretorioRemoto):
        return diretorioRemoto.replace(self.dirRemotoRaiz, self.dirLocalRaiz)
  
    def existeArquivoIndice(self):
        try:
            if self.sftp.stat(self.indice_md5):
                return True
        except:
            return False
  
    def recuperaMd5(self, entrada):
        try:
            md5 = self.dicionario_md5[entrada]
            return md5
        except KeyError:
            return None
  
    def atualizarIndice(self):
        for key in self.dicionario_md5.keys():
            if not os.path.exists(self.getDiretorioLocalFromRemoto(key)):
                self.dicionario_md5.pop(key)
  
    def carregarDicionarioMd5(self):
        if self.existeArquivoIndice():
            indice_local = self.getDiretorioLocalFromRemoto(self.indice_md5)
            self.sftp.get(self.indice_md5, indice_local)
            indice = open(indice_local, 'rb')
            self.dicionario_md5 = pickle.load(indice)
            indice.close()
  
    def salvarDicionarioMd5(self):
        indice_local = self.getDiretorioLocalFromRemoto(self.indice_md5)
        indice = open(indice_local, 'wb')
        pickle.dump(self.dicionario_md5, indice)
        indice.close()  
        self.copiarArquivoParaServidor(indice_local, self.indice_md5)
    
    def isArquivosIguaisMD5(self, local_file, remote_file):
        m_local = hashlib.md5()
        m_local.update(open(local_file, "rb").read())
        md5Local = m_local.digest()
        md5Remoto = self.recuperaMd5(remote_file)
        if md5Local == md5Remoto:
            return True
        else:
            return md5Local
  
    def calcularMd5(self, local_file):
        m_local = hashlib.md5()
        m_local.update(open(local_file, "rb").read())
        return m_local.digest()        
  
    def acompanharTransferenciaArquivo(self, tamanhoTransferido, tamanhoTotal):
        try:
            porcentagem = int((float(tamanhoTransferido) / float(tamanhoTotal)) * 100)
            pontos = porcentagem - self.statusTransferencia
            if pontos > 0:
                self.statusTransferencia = self.statusTransferencia + pontos
                sys.stdout.write(pontos * '.')
        except Exception, e:
            print '*** Exceção Lançada: %s: %s' % (e.__class__, e)
  
    def copiarArquivoParaServidor(self, local_file, remote_file):
        tentativas = 0
        try:
            self.statusTransferencia = 0
            self.sftp.put(local_file, remote_file, self.acompanharTransferenciaArquivo)
            print ''
        except:
            tentativas += 1     
            print 'ERRO ao enviar o arquivo ', local_file
            self.copiarArquivoParaServidor(self, local_file, remote_file)
        return tentativas
  
    def sincronizarPastas(self, dirLocal, dirRemoto):
        self.totais = {}
        self.pastaOrigem = dirLocal
        self.pastaDestino = dirRemoto
        self.getConexaoSSH()
        self.carregarDicionarioMd5()
        self.executarCopia()
        self.atualizarIndice()
        print 'Copiando o indice md5: ', 
        self.salvarDicionarioMd5()
        print 'Copiando o index.wsgi: ', 
        self.copiarArquivoParaServidor(self.getDiretorioLocalFromRemoto(self.wsgi), self.wsgi)
        self.transport.close()
    
    def deletarRecursosRemotos(self, dirRemoto):
        try:
            print 'PROCESSANDO A PASTA REMOTA - ', dirRemoto
            for entrada in  self.sftp.listdir(dirRemoto):
                remote_entry = self.pathJoin(dirRemoto, entrada)
                remote_entry = remote_entry.replace('\\','/')
                if self.isRemoteDir(remote_entry):
                    self.deletarRecursosRemotos(remote_entry)
                elif self.isRemoteFile(remote_entry):
                    local_file = self.getDiretorioLocalFromRemoto(remote_entry)
                    if not os.path.exists(local_file):
                        print "    (arquivo removido):", remote_entry, " (" + self.formataTamanhoArquivo(self.sftp.stat(remote_entry).st_size) + ") "
                        self.sftp.remove(remote_entry)
                        self.contabilizarTotais('arquivos_removidos', 1)
            if self.sftp.listdir != '' and not os.path.exists(self.getDiretorioLocalFromRemoto(dirRemoto)):
                print '    (diretório removido): ', dirRemoto
                self.sftp.rmdir(dirRemoto)
                self.contabilizarTotais('diretorios_removidos', 1)
        except Exception, e:
            print '*** Exceção Lançada ao deletar Recursos Remotos: %s: %s' % (e.__class__, e)
            sys.exit()
  
    def isRemoteDir (self, remote_path):
        try:
            st = self.sftp.stat( remote_path )
            return S_ISDIR(st.st_mode)
        except Exception:
            return False

    def isRemoteFile (self, remote_path):
        try:
            st = self.sftp.stat( remote_path )
            return S_ISREG(st.st_mode)
        except Exception:
            return False
  
    def formataTamanhoArquivo(self, tamanho):
        tipo = 1
        while(tamanho > 1024):
            tamanho = float(tamanho) / 1024.0
            tipo += 1
        if(tipo == 1):
            tamanho = "%.2f bytes" % (tamanho)
        elif(tipo == 2):
            tamanho = "%.2f Kb" % (tamanho)
        elif(tipo == 3):
            tamanho = "%.2f Mb" % (tamanho)
        elif(tipo == 4):
            tamanho = "%.2f Gb" % (tamanho)
        return tamanho.replace(".00", "")
          
    def contabilizarTotais(self, tipo, valor):
        self.totais[tipo] = self.getResultadoTotal(tipo) + valor
    
    def getResultadoTotal(self, chave):
        try:
            return self.totais[chave]
        except:
            return 0
  
    def executarCopia(self):
        print '=' * 60
        print 'Local = ' + self.pastaOrigem
        print 'Remoto = ' + self.pastaDestino
        print '=' * 60

        try:
            diretorioRemoto = self.pastaDestino
            self.contabilizarTotais('diretorios_criados', self.criarDiretorioRemoto(diretorioRemoto))
            for raiz, diretorios, arquivos in os.walk(self.pastaOrigem):
                raiz = raiz.replace('\\', '/')
                print 'PROCESSANDO A PASTA LOCAL - ', self.pathJoin(self.pastaOrigem, raiz)
                for diretorio in diretorios:
                    self.contabilizarTotais('diretorios', 1)
                    diretorioRemoto = self.getDiretorioRemotoFromLocal(self.pathJoin(raiz, diretorio))
                    self.contabilizarTotais('diretorios_criados', self.criarDiretorioRemoto(diretorioRemoto))
                for arquivo in arquivos:
                    self.contabilizarTotais('arquivos', 1)
                    local_file = self.pathJoin(raiz, arquivo)
                    remote_file = self.getDiretorioRemotoFromLocal(local_file)
                    is_up_to_date = False
                    try:
                        # verifica se o arquivo remoto existe
                        if self.sftp.stat(remote_file):
                            md5 = self.isArquivosIguaisMD5(local_file, remote_file)
                            if md5 == True:
                                print "    (não modificado):", arquivo + " (" + self.formataTamanhoArquivo(os.path.getsize(local_file)) + ")"
                                self.contabilizarTotais('arquivos_naomodificados', 1)
                                is_up_to_date = True
                            else:
                                print "    (modificado):", arquivo + " (" + self.formataTamanhoArquivo(os.path.getsize(local_file)) + ") ",
                                self.dicionario_md5[remote_file] = md5
                                self.contabilizarTotais('arquivos_modificados', 1)
                    except:
                        print "    (novo):", arquivo + " (" + self.formataTamanhoArquivo(os.path.getsize(local_file)) + ") ",
                        self.contabilizarTotais('arquivos_novos', 1)
                        md5 = self.calcularMd5(local_file)
                        self.dicionario_md5[remote_file] = md5
                    if not is_up_to_date:
                        self.contabilizarTotais('tentativas', self.copiarArquivoParaServidor(local_file, remote_file))
                        
            self.deletarRecursosRemotos(self.pastaDestino)
            
        except Exception, e:
            print '*** Exceção Lançada ao copiar arquivo: %s: %s' % (e.__class__, e)
            sys.exit()
        print '=' * 60
        print 'Número de tentativas para erro (retry):', self.getResultadoTotal('tentativas')
        print 'Total de diretórios criados:', self.getResultadoTotal('diretorios_criados')
        print 'Total de arquivos novos:', self.getResultadoTotal('arquivos_novos')
        print 'Total de arquivos modificados:', self.getResultadoTotal('arquivos_modificados')
        print 'Total de arquivos não modificados:', self.getResultadoTotal('arquivos_naomodificados')
        print 'Total de diretórios remotos removidos:', self.getResultadoTotal('diretorios_removidos')
        print 'Total de arquivos remotos removidos:', self.getResultadoTotal('arquivos_removidos')
        print 'Total de diretórios:', self.getResultadoTotal('diretorios')
        print 'Total de arquivos:', self.getResultadoTotal('arquivos')
        print 'Completo!'
        print '=' * 60

Preste atenção ao parâmetro self.caminhoArquivoConfiguracao, setando para o diretório onde se encontra o seu arquivo de configuração (settings.ini)

Exemplo de utilização (Chamada do Deploy):

#! /usr/bin/python
# -*- coding: iso-8859-1 -*-

from julianajabur.deploy import ajustarFlex, SFTPLocaweb

sftpLocaweb = SFTPLocaweb.SFTPLocaweb()

dir_local_wsgi='/app/pythonProjects/wsgi_apps/julianajabur'
dir_remote_wsgi = "/home/storage/abc/julianajabur/wsgi_apps/julianajabur"
sftpLocaweb.sincronizarPastas(dir_local_wsgi, dir_remote_wsgi)

print '\n\n\n\n'

dir_local_public='/app/pythonProjects/public_html/julianajabur'
dir_remote_public = "/home/storage/abc/julianajabur/public_html/julianajabur"
sftpLocaweb.sincronizarPastas(dir_local_public, dir_remote_public)

E por último segue o resultado da execução do script (SysOut), impresso na linha de comando:

Estabelecendo conexão com:  servidor.com 22 ...
============================================================
Local = /app/pythonProjects/wsgi_apps/julianajabur
Remoto = /home/storage/abc/julianajabur/wsgi_apps/julianajabur
============================================================
    (diretório já existe)  /home/storage/abc/julianajabur/wsgi_apps/julianajabur
PROCESSANDO A PASTA LOCAL -  /app/pythonProjects/wsgi_apps/julianajabur
    (diretório já existe)  /home/storage/abc/julianajabur/wsgi_apps/julianajabur/.settings
    (diretório já existe)  /home/storage/abc/julianajabur/wsgi_apps/julianajabur/julianajabur
    (não modificado): SFTPLocaweb.py (11.93 Kb)
    (não modificado): .project (422 bytes)
    (não modificado): .pydevproject (670 bytes)
PROCESSANDO A PASTA LOCAL -  /app/pythonProjects/wsgi_apps/julianajabur/.settings
    (não modificado): org.eclipse.ltk.core.refactoring.prefs (134 bytes)
    (não modificado): org.eclipse.core.resources.prefs (276 bytes)
PROCESSANDO A PASTA LOCAL -  /app/pythonProjects/wsgi_apps/julianajabur/julianajabur
    (diretório já existe)  /home/storage/abc/julianajabur/wsgi_apps/julianajabur/julianajabur/julianajaburapp
    (diretório já existe)  /home/storage/abc/julianajabur/wsgi_apps/julianajabur/julianajabur/deploy
    (não modificado): urls.py (650 bytes)
    (não modificado): urls.pyc (1004 bytes)
    (não modificado): __init__.pyc (141 bytes)
    (não modificado): __init__.py (0 bytes)
    (não modificado): settings.pyc (3.04 Kb)
    (não modificado): manage.py (654 bytes)
    (não modificado): amfgateway.pyc (1.46 Kb)
    (não modificado): amfgateway.py (855 bytes)
    (não modificado): settings.py (3.57 Kb)
PROCESSANDO A PASTA LOCAL -  /app/pythonProjects/wsgi_apps/julianajabur/julianajabur/julianajaburapp
    (não modificado): models.pyc (751 bytes)
    (não modificado): __init__.pyc (157 bytes)
    (não modificado): views.py (538 bytes)
    (não modificado): models.py (212 bytes)
    (não modificado): __init__.py (0 bytes)
    (não modificado): views.pyc (1.11 Kb)
    (não modificado): tests.py (539 bytes)
    (não modificado): admin.pyc (336 bytes)
    (não modificado): admin.py (122 bytes)
PROCESSANDO A PASTA LOCAL -  /app/pythonProjects/wsgi_apps/julianajabur/julianajabur/deploy
    (não modificado): __init__.pyc (160 bytes)
    (não modificado): __init__.py (0 bytes)
    (não modificado): SFTPLocaweb.py (11.07 Kb)
    (não modificado): deployLocaweb.py (600 bytes)
    (não modificado): SFTPLocaweb.pyc (11.45 Kb)
    (não modificado): ajustarFlex.py (1.19 Kb)
    (não modificado): ajustarFlex.pyc (1.90 Kb)
PROCESSANDO A PASTA REMOTA -  /home/storage/abc/julianajabur/wsgi_apps/julianajabur
PROCESSANDO A PASTA REMOTA -  /home/storage/abc/julianajabur/wsgi_apps/julianajabur/.settings
PROCESSANDO A PASTA REMOTA -  /home/storage/abc/julianajabur/wsgi_apps/julianajabur/julianajabur
PROCESSANDO A PASTA REMOTA -  /home/storage/abc/julianajabur/wsgi_apps/julianajabur/julianajabur/julianajaburapp
PROCESSANDO A PASTA REMOTA -  /home/storage/abc/julianajabur/wsgi_apps/julianajabur/julianajabur/deploy
============================================================
Número de tentativas para erro (retry): 0
Total de diretórios criados: 0
Total de arquivos novos: 0
Total de arquivos modificados: 0
Total de arquivos não modificados: 30
Total de diretórios remotos removidos: 0
Total de arquivos remotos removidos: 0
Total de diretórios: 4
Total de arquivos: 30
Completo!
============================================================
Copiando o indice md5: ....................................................................................................
Copiando o index.wsgi: ....................................................................................................

Até a próxima pessoal, espero que este script ajude muitas pessoas, qualquer dúvida, basta deixar um comentário aqui no blog e eu respondo.

Sintam-se à vontade para manipular o script como vocês bem entenderem, e se melhorarem ele, me enviem uma cópia com as melhorias para eu postar aqui no blog, …

Abraços,
Victor Jabur