12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491 |
- Download Python Documentation - https://docs.python.org/3/download.html
- Python core development news and information - https://blog.python.org/
- Python Developer's Guide - https://devguide.python.org/
- Python best practices for formatting code - https://www.python.org/dev/peps/pep-0008/
- Installing packages using pip and virtual environments -
- packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/
- Packaging python projects -
- https://packaging.python.org/en/latest/tutorials/packaging-projects/
- pyenv Tutorial -
- https://amaral.northwestern.edu/resources/guides/pyenv-tutorial
- Pythontic.com Pythonic Permutations
- Scipy Lecture Notes - https://scipy-lectures.org/
- Book: Python Scripting for Computational Science
- Book: Think Python
- https://southampton.ac.uk/~fangohr/training/python/pdfs/
- PLOS - Computational Biology
- https://www.physics.ohio-state.edu/undergrad/greStuff/
- https://docs.python.org/3/tutorial/venv.html # tutorial for venv
- Boston University TechWeb -
- www.bu.edu/tech/support/research/software-and-programming/common-languages/python/
- Google Python Style Guide - https://google.github.io/styleguide/pyguide.html
- Python charts - https://python-charts.com/
- PyQt Scientific Graphics and GUI Library for Python - https://www.pyqtgraph.org
- The complete PyQt5 tutorial - https://www.pythonguis.com/pyqt5-tutorial/
- Customizing Ticks -
- https://jakevdp.github.io/PythonDataScienceHandbook/04/10-customising-ticks.html
- Python Concurrency Learning Paths - https://superfastpython.com/learning-paths/
- gmpy2 - https://gmpy2.readthedocs.io/en/latest/intro.html
- Memory profiler - https://github.com/pythonprofilers/memory_profiler
- Meliae - https://launchpad.net/meliae
- Built-in Types - https://docs.python.org/3/library/stdtypes.html#int.bit_length
- Pytest - https://docs.pytest.org/en/7.4.x/contents.html
- ISciNumPy - https://iscinumpy.dev/
- Software Engineering for Scientific Computing -
- https://henryiii.github.io/se-for-sci/content/intro.html
- Scientific Python - https://scientific-python.org/
- Scientific Python Library Development Guide -
- https://learn.scientific-python.org/development/
- Using pytest proto-fixtures - https://michaelgoerz.net/notes/using-proto-fixtures.html
- Python plot HTML browser - www.scivision.dev/python-html-plotting-iframe-share/
- Catch Numpy warnings as error - https://www.scivision.dev/python-numpy-catch-warnings/
- Parallel processing in Python -
- https://berkeley-scf.github.io/tutorial-parallelization/parallel-python
- Catch all errors in Python - https://embeddedinventor.com/python-catch-all-errors/
- Better Scientific Software (BSSw) - https://bssw.io/
- Python Packaging User Guide -
- https://packaging.python.org/en/latest/tutorials/packaging-projects/
- Programming is simply the act of entering instructions for the computer to perform.
- Programming instructions - source code
- Debugging programs - finding and fixing errors
- People develop programming skills through practice.
- Three Rules of Programming:
- 1. It Must Work!
- 2. It Must be Maintainable!
- 3. Now Think Performance.
- Python source files are UTF-8-encoded text files that normally have a .py
- suffix. International (Unicode) characters can be freely used in the source
- code as long as you use the UTF-8 encoding.
- It is common to use #! to specify the interpreter on the first line of a
- program, like this:
- #!/usr/bin/env python
- The interpreter runs statements in order until it reaches the end of the input
- file. At that point, the program terminates and Python exits.
- # package
- The difference between a regular directory and a Python package is that the
- latter includes a file named __init__.py that instructs Python's interpreter to
- understand the directory as a package with Python code.
- # dynamically typed language
- Python does not require a variable's data type to be specified upon initialization.
- # naming conventions
- * variable: snake_case
- * global constant: ALL_CAPS
- * function: snake_case
- * class: CamelCase
- * hidden/not public: _underscore
- * built-in: __dunder__
- # load a file into the python console
- $ python -i program.py
- # list of Python's built-ins that you should avoid using as variables
- >>> import builtins
- >>> dir(builtins)
- ['ArithmeticError', 'AssertionError', 'AttributeError',
- 'BaseException', 'BaseExceptionGroup', 'BlockingIOError',
- 'BrokenPipeError', 'BufferError', 'BytesWarning',
- 'ChildProcessError', 'ConnectionAbortedError', 'ConnectionError',
- 'ConnectionRefusedError', 'ConnectionResetError', 'DeprecationWarning',
- 'EOFError', 'Ellipsis', 'EncodingWarning', 'EnvironmentError',
- 'Exception', 'ExceptionGroup', 'False', 'FileExistsError',
- 'FileNotFoundError', 'FloatingPointError', 'FutureWarning',
- 'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning',
- 'IndentationError', 'IndexError', 'InterruptedError', 'IsADirectoryError',
- 'KeyError', 'KeyboardInterrupt', 'LookupError', 'MemoryError',
- 'ModuleNotFoundError', 'NameError', 'None', 'NotADirectoryError',
- 'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError',
- 'PendingDeprecationWarning', 'PermissionError', 'ProcessLookupError',
- 'RecursionError', 'ReferenceError', 'ResourceWarning', 'RuntimeError',
- 'RuntimeWarning', 'StopAsyncIteration', 'StopIteration', 'SyntaxError',
- 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'TimeoutError',
- 'True', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
- 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
- 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning', 'ZeroDivisionError',
- '__build_class__', '__debug__', '__doc__', '__import__', '__loader__', '__name__',
- '__package__', '__spec__', 'abs', 'aiter', 'all', 'anext', 'any', 'ascii', 'bin',
- 'bool', 'breakpoint', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod',
- 'compile', 'complex', 'copyright', 'credits', 'delattr', 'dict', 'dir', 'divmod',
- 'enumerate', 'eval', 'exec', 'exit', 'filter', 'float', 'format', 'frozenset',
- 'getattr', 'globals', 'hasattr', 'hash', 'help', 'hex', 'id', 'input', 'int',
- 'isinstance', 'issubclass', 'iter', 'len', 'license', 'list', 'locals', 'map',
- 'max', 'memoryview', 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print',
- 'property', 'quit', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
- 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'vars', 'zip']
- >>> dir(__builtins__)
- ['ArithmeticError', 'AssertionError', 'AttributeError',
- 'BaseException', 'BaseExceptionGroup', 'BlockingIOError',
- 'BrokenPipeError', 'BufferError', 'BytesWarning',
- 'ChildProcessError', 'ConnectionAbortedError', 'ConnectionError',
- 'ConnectionRefusedError', 'ConnectionResetError', 'DeprecationWarning',
- 'EOFError', 'Ellipsis', 'EncodingWarning', 'EnvironmentError',
- 'Exception', 'ExceptionGroup', 'False', 'FileExistsError',
- 'FileNotFoundError', 'FloatingPointError', 'FutureWarning',
- 'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning',
- 'IndentationError', 'IndexError', 'InterruptedError', 'IsADirectoryError',
- 'KeyError', 'KeyboardInterrupt', 'LookupError', 'MemoryError',
- 'ModuleNotFoundError', 'NameError', 'None', 'NotADirectoryError',
- 'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError',
- 'PendingDeprecationWarning', 'PermissionError', 'ProcessLookupError',
- 'RecursionError', 'ReferenceError', 'ResourceWarning', 'RuntimeError',
- 'RuntimeWarning', 'StopAsyncIteration', 'StopIteration', 'SyntaxError',
- 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'TimeoutError',
- 'True', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
- 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
- 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning', 'ZeroDivisionError',
- '__build_class__', '__debug__', '__doc__', '__import__', '__loader__', '__name__',
- '__package__', '__spec__', 'abs', 'aiter', 'all', 'anext', 'any', 'ascii', 'bin',
- 'bool', 'breakpoint', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod',
- 'compile', 'complex', 'copyright', 'credits', 'delattr', 'dict', 'dir', 'divmod',
- 'enumerate', 'eval', 'exec', 'exit', 'filter', 'float', 'format', 'frozenset',
- 'getattr', 'globals', 'hasattr', 'hash', 'help', 'hex', 'id', 'input', 'int',
- 'isinstance', 'issubclass', 'iter', 'len', 'license', 'list', 'locals', 'map',
- 'max', 'memoryview', 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print',
- 'property', 'quit', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
- 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'vars', 'zip']
- >>> len(dir(__builtins__))
- 158
- >>> type(dir(__builtins__))
- <class 'list'>
- >>> dir(__builtins__)[0]
- 'ArithmeticError'
- >>> type(dir(__builtins__)[0])
- <class 'str'>
- >>> [s for s in dir(__builtins__) if s.islower() and not s.startswith('_')]
- ['abs', 'aiter', 'all', 'anext', 'any', 'ascii', 'bin', 'bool', 'breakpoint',
- 'bytearray', 'bytes', 'callable', 'chr', 'classmethod', 'compile', 'complex',
- 'copyright', 'credits', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval',
- 'exec', 'exit', 'filter', 'float', 'format', 'frozenset', 'getattr', 'globals',
- 'hasattr', 'hash', 'help', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass',
- 'iter', 'len', 'license', 'list', 'locals', 'map', 'max', 'memoryview', 'min',
- 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', 'property', 'quit', 'range',
- 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod',
- 'str', 'sum', 'super', 'tuple', 'type', 'vars', 'zip']
- >>> [s for s in dir(__builtins__) if s.endswith('Error')]
- ['ArithmeticError', 'AssertionError', 'AttributeError', 'BlockingIOError',
- 'BrokenPipeError', 'BufferError', 'ChildProcessError', 'ConnectionAbortedError',
- 'ConnectionError', 'ConnectionRefusedError', 'ConnectionResetError', 'EOFError',
- 'EnvironmentError', 'FileExistsError', 'FileNotFoundError', 'FloatingPointError',
- 'IOError', 'ImportError', 'IndentationError', 'IndexError', 'InterruptedError',
- 'IsADirectoryError', 'KeyError', 'LookupError', 'MemoryError', 'ModuleNotFoundError',
- 'NameError', 'NotADirectoryError', 'NotImplementedError', 'OSError', 'OverflowError',
- 'PermissionError', 'ProcessLookupError', 'RecursionError', 'ReferenceError',
- 'RuntimeError', 'SyntaxError', 'SystemError', 'TabError', 'TimeoutError', 'TypeError',
- 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError',
- 'UnicodeTranslateError', 'ValueError', 'ZeroDivisionError']
- >>> [s for s in dir(__builtins__) if s.endswith('Warning')]
- ['BytesWarning', 'DeprecationWarning', 'EncodingWarning', 'FutureWarning',
- 'ImportWarning', 'PendingDeprecationWarning', 'ResourceWarning', 'RuntimeWarning',
- 'SyntaxWarning', 'UnicodeWarning', 'UserWarning', 'Warning']
- >>> 'len' in dir(__builtins__)
- True
- >>> len('string')
- 6
- >>> builtins.len('string') # built-in len() function as builtins.len()
- 6
- >>> builtins.len
- <built-in function len>
- >>> builtins.len is len
- True
- >>> __builtins__.len('string') # __builtins__ module without importing it
- 6
- >>> __builtins__.len
- <built-in function len>
- >>> __builtins__.len is len
- True
- # install the build tools
- $ doas apt-get update
- $ doas apt-get install make build-essential libssl-dev zlib1g-dev libbz2-dev
- libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev
- xz-utils tk-dev libffi-dev liblzma-dev libgdbm-compat-dev python3-openssl
- # Python-3.11.4 Current Stable version installation
- $ wget https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz
- $ tar -xzvf Python-3.11.4.tgz
- $ mv Python-3.11.4/ ~/.python/
- $ cd .python/
- $ ./configure --enable-optimizations
- $ make
- $ make test
- $ doas make altinstall
- Note: The default Python installation is /usr/bin. If you want to install Python
- under /usr/local/bin instead of overwriting the default, do this:
- $ doas make altinstall
- # upgrade Python-3.11.4 to Python-3.11.5 [https://www.python.org/downloads/source/]
- 1. download current stable release: Python 3.11.5 [Gzipped source tarball]
- $ wget https://www.python.org/ftp/python/3.11.5/Python-3.11.5.tgz
- 2. untar Python-3.11.5.tgz
- $ tar -xzvf Python-3.11.5.tgz
- 3. move Python-3.11.5/ to ~/.python3.11.5/
- $ mv downloads/Python-3.11.5/ .python3.11.5/
- 4. change directory .python3.11.5/
- $ cd .python3.11.5/
- 5. read the file README.rst
- $ vi -M README.rst
- 6. build instructions on linux
- $ ./configure --enable-optimizations
- $ make
- $ make test
- $ doas make altinstall
- 7. remove Python 3.11.4 version ~/.python/
- $ doas rm -rf ~/.python/
- 8. move ~/.python3.11.5/ ~/.python/
- $ mv .python3.11.5/ .python/
- 9. verify Python 3.11.5 install path
- $ which python3.11
- 10. verify Python version
- $ python3.11 --version
- # built-in type hinting/variable annotation
- You can add type hinting with the following built-in types:
- * int integer
- * float floating point number
- * bool boolean value (subclass of int)
- * str text, sequence of unicode codepoints
- * bytes 8-bit string, sequence of byte values
- * object an arbitrary object (object is the common base class)
- These can be used both in functions and in variable annotation. The concept of
- variable annotation was added to the Python language in 3.6 Variable annotation
- allows you to add type hints to variables.
- Here are some examples:
- x: int # a variable named x without initialization
- y: float = 1.0 # a float variable, initialized to 1.0
- z: bool = False
- a: str = "Hello type hinting"
- # sequence type hinting
- A collection is a group of items in Python. Common collections or sequences are
- list, dict, tuple and set. However, you cannot annotate variables using these
- built-in types. Instead, you must use the typing module.
- from typing import List
- names: List[str] = ["Math"]
- Here you created a list with a single str in it. This specifies that you are
- creating a list of strings. If you know the list is always going to be the same
- size, you can specify each item's type in the list:
- from typing import List
- names: List[str, str] = ["Math", "Physics"]
- Hinting tuples is very similar:
- from typing import Tuple
- s: Tuple[int, float, str] = (5, 3.14, "laplace")
- Dictionaries are a little different in that you should hint types the key and
- values are:
- from typing import Dict
- d: Dict[str, int] = {"Newton": 25}
- If you know a collection will have variable size, you can use an ellipses:
- from typing import Tuple
- t: Tuple[int, ...] = (4, 5, 6)
- # hinting values that could be None
- Sometimes a value needs to be initialized as None, but when it gets set later,
- you want it to be something else.
- For that, you can use Optional:
- from typing import Optional
- result: Optional[str] = function()
- On the other hand, if the value can never be None, you should add an assert to
- your code:
- assert result is not None
- # type hinting functions
- Type hinting functions is similar to type hinting variables. The main
- difference is that you can also add a return type to a function.
- Let's take a look at an example:
- def adder(x: int, y: int) -> None:
- print(f"The total of {x} + {y} = {x+y}")
- This example shows you that adder() takes two arguments, x and y, and that they
- should both be integers. The return type is None, which you specify using the
- -> after the ending parentheses but before the colon.
- Let's say that you want ot assign the adder() function to a variable. You can
- anootate the variable as a Callable like this:
- from typing import Callable
- def adder(x: int, y: int) -> None:
- print(f"The total of {x} + {y} = {x+y}")
- a: Callable[[int, int], None] = adder
- The Callable takes in a list of arguments for the function. It also allows you
- to specify the return type.
- Let's look at one more example where you pass in more complex arguments:
- from typing import Tuple, Optional
- def some_func(x: int, y: Tuple[str, str], z: Optional[float]: = None): -> Optional[str]:
- if x > 10:
- return None
- return "You called some_func"
- For this example, you created some_func() that accepts 3 arguments:
- * an int
- * a two-item tuple of strings
- * an optional float that is defaulted to None
- Note that when you use defaults in a function, you should add a space before
- and after the equals sign when using type hints.
- It also returns either None or a string.
- # when things get complicated
- You have already learned what to do when a value can be None, but what else can
- you do when things get complicated? For example, what do you do if the argument
- being passed in can be multiple different types?
- For that specific use case, you can use Union:
- from typing import Union
- z: Union[str, int]
- What this type hint means is that the variable z, can be either a string or an
- integer.
- There are also cases where a function may take in an object. If that object can
- be one of several different objects, then you can use Any.
- x: Any = some_function()
- Use Any with caution because you can't really tell what it is that you are
- returning. Since it can be "any" type, it is like catching all exceptions with
- a bare except. You don't know what exception you are catching with that and you
- also don't know what type you are hinting at when you use Any.
- # -m flag
- The -m command-line flag will import any Python module and run it as a script.
- # debug mode
- $ python -m pdb cachefib.py
- >/home/user/directory/program.py(3)<module>()
- (Pdb)
- (Pdb) ll
- 1 #!/usr/bin/env python
- 2
- 3 cache = {}
- 4
- 5 def fibonacci(n):
- 6 if n < 3:
- 7 return 1
- 8
- 9 if n in cache:
- 10 return cache[n]
- 11
- 12 cache[n] = fibonacci(n - 1) + fibonacci(n - 2)
- 13 return cache[n]
- 14
- 15
- 16 fib = fibonacci(30)
- 17 -> print(f"The 30th Fibonacci number is {fib}")
- (Pdb)
- (Pdb) whatis fib
- <class 'int'>
- (Pdb) pp fib # pretty print
- 832040
- (Pdb) p fibonacci(50)
- 12586269025
- (Pdb) b cachefib.py:13 # breakpoint at line 13
- (Pdb) b cachefib.fibonacci # breakpoint at function fibonacci
- (Pdb) b 24 # breakpoint at line 24
- (Pdb) break # list of breakpoints
- (Pdb) disable 1 # disable a breakpoint
- (Pdb) enable 1 # enable a breakpoint
- (Pdb) clear 1 # remove a breakpoint entirely
- (Pdb) tbreak 3 # temporary break at line 3
- (Pdb) l # list 11 lines around the current line
- (Pdb) l 11,21 # list the lines 11 - 21
- (Pdb) ll # list all the lines in the program
- (Pdb) exit # to exit debug mode
- Table of pdb commands:
- args [a]
- Print the argument list of the current function
- break [b]
- Creates a breakpoint (requires parameters) in the program execution
- By using the break command to set breakpoints, you'll run the program
- up until the specified breakpoint.
- Type clear and then y to remove all current breakpoints. You can then
- place a breakpoint where a function is defined.
- To see a list of breakpoints that are currently set to run, use the
- command break without any arguments. You'll receive information about
- the particularities of the breakpoint(s) you've set.
- We can also disable a breakpoint with the command disable and the
- number of the breakpoint.
- To enable a breakpoint, use the enable command, and to remove a breakpoint
- entirely, use the clear command.
- Creating temporary breakpoints that are automatically cleared the first
- time program execution hits the point with the command tbreak.
- continue [c/cont]
- Continues program execution until there is a breakpoint
- help [h]
- Provides list of commands or help for a specified command
- You can use the command help to learn pdf commands, and
- help <command> to learn more about a specific command.
- jump [j]
- Set the next line to be executed
- list [l]
- Print the source code around the current line
- Without providing arguments, the list command provides 11 lines
- around the current line.
- list 3,7 [l 3,7]
- Print the lines 3-7
- longlist [ll]
- List all source code for the current function or frame
- next [n]
- Continue execution until the next line in the current function is reached
- run
- Restart the debugged python program at any place within the program
- step [s]
- Execute the current line, stopping at first possible occasion
- The difference between step and next is that step will stop within a called
- function, while next executes called functions to only stop at the next line
- of the current function.
- pp
- Pretty-prints the value of the expression using the pprint module
- quit/exit [q]
- Aborts the program
- return [r]
- Continue execution until the current function returns
- Note: Call the last command you called by pressing the <Enter> key at the prompt.
- Ref:
- https://docs.python.org/3/library/pdb.html
- https://www.digitalocean.com/community/tutorials/how-to-use-the-python-debugger
- https://www.redhat.com/sysadmin/python-debugger-pdb
- # linux system packages required for scipy
- sudo apt-get install gcc gfortran python3-dev meson libopenblas-dev
- liblapack-dev cython
- # wisdom of OOP
- write methods only when the exclusive association with the data type is not in
- doubt (e.g. ADT's).
- # executing python script in ipython
- %run script.py
- Terminate an interactive session:
- >>> import sys
- >>> sys.exit()
- or
- >>> raise SystemExit
- or
- control-D for UNIX/Linux
- Shell commands in Python:
- >>> import os
- >>> os.system('ls -CF')
- In Python interpreters, the underscore character _ is a variable with the value
- of the previous command's output.
- Functions are defined with parameters and are called with arguments.
- When defining a function, its input variables are called the parameters of the
- function. The input used when executing the function is called its argument.
- A list is a basic Python data structure.
- To see the methods associated with a list, type the object name (list),
- followed by a period, and press tab.
- The help() function is the key to understanding most other functions.
- >>> help(round) # display description of what the function does
- >>> help(round(-2.01)) # Error
- Python evaluates an expression like this from the inside out. First it
- calculates the value of round(-2.01), then it provides help on the output of
- that expression.
- >>> help() # to start interactive help utility
- help> list # get documentation on the list class
- help> numpy # get documentation on the numpy package
- help> scipy # get documentation on the scipy package
- help> matplotlib # get documentation on the matplotlib package
- help> matplotlib.pylab # get documentation on the matplotlib.pylab
- help> matplotlib.pyplot # get documentation on the matplotlib.pyplot
- help> quit/q # end the interactive help session
- Strings are a collection of characters which are stored together to represent
- arbitrary text inside a python program.
- Special characters in strings:
- \ continuation
- \\ literal backslash
- \' single quote
- \" double quote
- \a bell
- \b backspace
- \e escape character
- \0 null terminator
- \n newline
- \t horizontal tab
- \f form feed
- \r carriage return
- \OXX octal character XX
- \xXX hexadecimal value XX
- threelines = 'First\ threelines = '''First
- Second\ Second
- Third' Third'''
- """ """ or ''' ''', creates a multi-line string literal
- Python provides what are called raw strings, in which the character sequences
- have no special meaning. To construct a raw string, precede the opening quote
- character with either a lowercase or uppercase (r or R). Note, however that a
- backslash cannot be the very last character of a raw string.
- >>> print('Here is a backslash: \\')
- Here is a backslash: \
- >>> print(r'Here is a backslash: \ ')
- Here is a backslash: \
- Unicode string:
- Unicode string characters are stored in 16 bits instead of the 8 bits used by
- a normal string.
- print(u'institute of mathematical sciences')
- If a unicode string is combined with a regular string, the resulting string
- will also be a Unicode string.
- Python considers the type of an object when it tries to apply an operator, so
- that if you try to concatenate a string and a number, you'll have problems.
- >>> x = 12./7.
- >>> print("The answer is " + x)
- The number (x) must first be converted to a string before it can be
- concatenated. Python provides two ways to do this: the core function repr, or
- the backquote operator ('').
- >>> print("The answer is " + repr(x))
- The answer is 1.71428571429
- >>> print("The answer is " + 'x')
- The answer is 1.71428571429
- The asterisk (*), when used between a string and an integer creates a new
- string with the old string repeated by the value of the integer. The order of
- the arguments not important.
- >>> '-' * 10
- ----------
- >>> 10 * '-'
- ----------
- indexing and slicing string:
- >>> name = 'institute'
- >>> for n in range(len(name)):
- print(name[n])
- i
- n
- s
- t
- i
- t
- u
- t
- e
- >>> name = 'institute'
- >>> for n in name:
- print(n)
- i
- n
- s
- t
- i
- t
- u
- t
- e
- >>> name = 'institute of mathematical sciences'
- >>> name[0]
- i
- >>> name[-1]
- s
- >>> len(name)
- 34
- >>> name[:50]
- institute of mathematical sciences
- >>> name[-50:]
- institute of mathematical sciences
- If you use a value for a slice index which is larger than the length of the
- string, python does not raise an exception, but treats the index as if it was
- the length of the string.
- Using a second index which is less than or equal to the first index will result
- in an empty string.
- Strings in python are immutable objects, this means that you can't change the
- value of a string in place.
- >>> name = 'institute of mathematical sciences'
- >>> name.split()
- ['institute', 'of', 'mathematical', 'sciences']
- >>> name = 'institute of mathematical sciences'
- name.split(' ')
- ['institute', 'of', 'mathematical', '', 'sciences']
- # Python Key words
- and as assert break class continue def del elif
- else except exec finally for from global if import in is
- lambda not or pass print raise return try while with yield
- (Note: In Py.3.0, exec is no longer a keyword)
- Arithmetic operations:
- Operator Name Description
- a + b Addition Sum of a and b
- a - b Subtraction Difference of a and b
- a * b Multiplication Product of a and b
- a / b True division Quotient of a and b
- a // b Floor division Quotient of a and b, removing fractional parts
- a % b Modulus Integer remainder after division of a by b
- a ** b Exponentiation a raised to the power of b
- -a Negation The negative of a
- +a Unary plus
- * The division operator (/) produces a floating-point number when applied to
- integers. Therefore, 7/4 is 1.75.
- * The truncating division operator //, also known as floor division, truncates
- the result to an integer and works with both integers and floating-point numbers.
- * The modulo operator returns the remainder of the division x//y.
- For example, 7 % 4 is 3.
- * For floating-point numbers, the modulo operator returns the floating-point
- remainder of x//y, which is x - (x//y) * y
- Mathematic Functions:
- Function Description
- abs(x) Absolute value
- divmod(x, y) Returns (x//y, x%y)
- pow(x, y) Returns (x**y)
- pow(x, y, z) Returns (x**y)%z
- round(x, n) Rounds to the nearest multiple of 10 to the nth power
- * round() function implements "banker's rounding". If the value being rounded is
- equally close to two multiples, it is rounded to the nearest even multiple.
- For example, 0.5 is rounded to 0.0 and 1.5 is rounded to 2.0
- Bit Manipulation Operators:
- Operation Description
- x << y Left shift
- x >> y Right shift
- x & y Bitwise and
- x | y Bitwise or
- x ^ y Bitwise xor (exclusive or)
- ~x Bitwise negation
- * One would commonly use these with binary integers.
- For example:
- a = 0b11001001
- mask = 0b11110000
- x = (a & mask) >> 4 # x = 0b1100 (12)
- Comparison Operators:
- Operation Description
- x == y Equal to
- x != y Not equal to
- x < y Less than
- x > y Greater than
- x >= y Greater than or equal to
- x <= y Less than or equal to
- * The result of a comparison is a Boolean value True or False
- * A value is considered false if it is literally False, None, numerically zero,
- or empty. Otherwise, it's considered true.
- Logical Operators:
- Operator Description
- x or y If x is false, return y; otherwise, return x
- x and y If x is false, return x; otherwise, return y
- not x If x is false, return True; otherwise, return False
- * Python does not have increment (++) or decrement(--) operators.
- pyenv Tutorial: https://amaral.northwestern.edu/resources/guides/pyenv-tutorial
- $ git clone git@github.com:pyenv/pyenv.git .pyenv
- $ vi .bashrc
- export PYENV_ROOT="$HOME/.pyenv"
- export PATH="$PYENV_ROOT/bin:$PATH"
- eval "$(pyenv init -)"
- $ source ~/.bashrc
- pyenv walkthrough:
- $ pyenv global
- system
- $ pyenv versions
- * system (set by /home/user/.pyenv/version)
- $ pyenv install -list # show all available python versions to install
- $ pyenv install 3.11.0 # install python version 3.11.0
- $ python versions # pyenv now lists two python versions
- * system (set by /home/user/.pyenv/version)
- 3.11.0
- $ pyenv global 3.11.0 # to use python 3.11.0 as the global
- # To use pyenv as projet-specific, or local version
- $ pyenv global system
- $ mkdir project_directory
- $ cd project_directory/
- project_directory$ pyenv local 3.11.0
- project_directory$ python -V
- Python 3.11.0
- project_directory$ cd ..
- $ python -V
- Python 3.9.2
- # delete/remove virtual environment with pyenv
- $ pyenv virtualenv-delete <name>
- check this: $ pyenv uninstall <name>
- Creating a virtual environment:
- In Python 3.6+, the recommended way to create a virtual environment is to run:
- $ python3 -m venv /path/to/new/virtual/environment
- Make sure that python3 resolves to whichever version of python3 you'd like to
- bind to your virtual environment. For example, to create a new virtual
- environment for CS41 named cs41-env in your home directory, you could run:
- $ python3 -m venv ~/cs41-env
- Activation and Deactivation:
- To activate a virtual environment on macOS or Linux running bash or zsh, source
- the following path:
- $ source ~/cs41-env/bin/activate
- https://github.com/stanfordpython/python-handouts/blob/master/virtual-environment.md
- $ deactivate # deactivate a virtual environment
- which python3 # installation search path of python3
- /usr/bin/python3 # in Ubuntu 16.04
- When a script file is used, it is sometimes useful to be able to run the script
- and enter interactive mode afterwards. This can be done by passing -i before
- the script.
- >>> python3 -i script.py
- !pydoc numpy # documentation for numpy in ipython
- Example:
- #!/usr/bin/env python3
- from math import sin
- import sys
- x = float(sys.argv[1])
- print('sin({0}) = {1}'.format(x, sin(x)))
- print('sin({x:q}) = {s:.3f}'.format(x=x, s=sin(x)))
- # curly brackets are place holders - {0}, {1}
- Print Statements:
- x = 0.8
- print("The value of x is", x)
- print("sin(%f) = %f" %(x, sin(x)))
- print("sin(%.2f) = %.2f" %(x, sin(x)))
- print("sin({0}) = {1}".format(x, sin(x)))
- print("sin({x:q}) = {s:.3f}".format(x=x, s=sin(x)))
- Complex Numbers:
- a = 1+2j
- b = 3-5j
- print(a*b)
- Logrithm:
- from math import log10
- log10(5)
- Special characters in strings:
- s = "\"This is a quote\" and \n here comes a backslash: \\"
- print(s)
- "This is a quote" and
- here comes a backslash: \
- String concatenation:
- Strings can be glued together with the + and the * operators
- "hello "*3 + "world"
- 'hello hello hello world'
- quote = 'I will not eat chips all day"
- (quote + ", ")*10 + quote
- "1"*10 = 1111111111
- int("1")*10 = 10
- Slicing:
- You can extract a sub-string with the [start:end] slicing notation:
- quote[2:6]
- 'will'
- If the start argument is left out, the substring will start from the first
- charachter:
- quote[:6] # I will
- quote[7:] # not eat chips all day
- Negative indices can be used to index "from the right":
- | c | h | i | p | s |
- 0 1 2 3 4 5
- -5 -4 -3 -2 -1 0
- 'chips'[1:-2]
- 'hi'
- Python strings ar
- import numpy as np
- dir(np)
- help(np.zeros)
- chmod u+x file.py
- ./file.py
- import this # Zen of Python
- # exit
- CTRL-D # to exit python
- quit()/exit()
- Pip (recursive acronym for "Pip installs Packages") is a cross platform package
- manager for installing and managing Python packages (which can be found in the
- Python Package Index (PyPI))
- # string formaat code
- %s string
- %c character
- %d integer
- %f floating pointer
- %o octal number
- %x hexadecimal number
- %e scientific notation
- # system useage
- python3 -m pip install <pkgname> # install
- python3 -m pip install --upgrade <pkgname> # upgrade
- python3 -m pip search <pkgname> # search
- python3 -m pip list | less -N # list
- python3 -m pip help | less -N # help
- pip3 --version # version
- # virtual environment using venv
- ## install
- pip install setuptools
- pip install wheel
- pip install numpy
- pip install scipy
- pip install sympy
- pip install gnuplotlib
- pip install matplotlib
- pip install matplotlib-venn
- pip install seaborn
- pip install altair
- pip install bokeh
- pip install dash
- pip install pandas
- pip install pylint
- pip install keras
- pip install theano
- pip install tensorflow
- pip install num2word
- pip install num2words
- pip install sklearn
- pip install imageio
- pip install mglearn
- pip install mpld3
- pip install mpmath
- pip install nltk
- pip install opencv-python
- pip install pygal
- pip install altair
- pip install umPlot
- pip install tilemapbase
- pip install PROJ
- pip install pydataset
- Ref: https://packaging.python.org/tutorials/installing-packages/
- Ref: https://pip.pypa.io/en/latest/ # pip documentation
- ## upgrade
- pip install --upgrade pip
- pip install --upgrade <pkgname>
- pip install --upgrade `pip list --outdated | awk 'NR>2 {print $1}'`
- for pip > 20.0:
- pip list --format freeze --outdated | sed 's/=.*//g' | xargs -n1 pip install -U
- ## downgrade
- pip install <pkgname>==<version>
- ## search
- pip search packagename
- ## show
- pip show packagename
- ## list
- pip list | less -N
- ## list pkgs [Documentation: https://pip.pypa.io/en/stable/]
- # list outdated packages
- pip list --outdated
- # list outdated packages using freeze formatting
- pip list --outdated --format=freeze
- # list uptodate packages
- pip list --uptodate
- # list editable projects
- pip list --editable
- # do not list globally-installed packages
- pip list --local
- # output packages installed in user-site
- pip list --user
- # restrict to the specified installation path
- pip list --path <path>
- # include pre-release and development versions
- pip list --pre
- # select the output format: columns, freeze, json
- pip list --format <list_format>
- # list packages that are not dependencies
- pip list --not-required
- # exclude editable package from output
- pip list --exclude-editable
- # include editable package from output
- pip list --include-editable
- # exclude specified package from the output
- pip list --exclude <package>
- # base URL of the python package index
- pip list --index-url <url>
- # extra URLs of package indexes to use
- pip list --extra-index-url <url>
- # ignore package index
- pip list --no-index
- # if a URL or path to an html file, then parse for links to archives such as
- # sdist (tar.gz) or wheel (.whl) files.
- pip list --find-links <url>
- for pip >= 10.0.1:
- import pkg_resources
- from subprocess import call
- packages = [dist.project_name for dist in pkg_resources.working_set]
- call("pip install --upgrade " + ' '.join(packages), shell=True)
- ## to upgrade all local packages
- pip install pip-review
- pip-review --local --interactive
- ## get complete information of installed package
- pip show <pkgname>
- ## help
- pip help | less
- (To see a list of all commands type)
- pip help install | less
- ## write all packages list to a file
- pip freeze > requirements.txt
- open the text file, replace the == with >=, and execute
- pip install -r requirements.txt --upgrade
- ## version
- pip --version
- ## uninstall
- pip uninstall <pkgname>
- pip uninstall <pkgname1> <pkgname2> <pkgname3>
- # pip
- Usage: sudo python3 -m pip <command> [options]
- Commands:
- install Install packages
- download Download packages
- uninstall Uninstall packages
- freeze Output installed packages in requirements format
- list List installed packages
- show Show information about installed packages
- check Verify installed packages have compatible dependencies
- search Search PyPI for packages
- wheel Build wheels from your requirements
- hash Compute hashes of package archives
- completion A helper command used for command completion
- help Show help for commands
- # to check installed
- python3 -c "import numpy"
- python3 -c "import sklearn"
- # pylint
- pylint file.py
- # pysparks
- pysparks file.py
- time = [time for time in np.linspace(0, 40, 10) for n in range(2)]
- print(time)
- [0.0, 0.0, ... 40.0, 40.0]
- time = [np.linspace(0, 40, 10) for n in range(2)]
- print(time)
- [array([0., ... 40.]), array([0., ... 40.])]
- time = 2*[np.linspace(0, 40, 10)] # result is same as above
- print(time)
- [array([0., ... 40.]), array([0., ... 40.])]
- # to check the architecture of python installed
- import platform, sys
- platform.architecture(), sys.maxsize
- import numpy as np
- np.__version__
- # for reference
- /usr/local/lib/python3.5/dist-packages/numpy/core/function_base.py
- # to find the python architecture
- import platform
- platform.architecture()
- Matplotlib gallery:
- https://matplotlib.org/gallery.html
- https://github.com/rasbt/matplotlib-gallery
- http://nbviewer.ipython.org/github/cs109/content/blob/master/
- lec_03_statistical_graphs.ipynb
- Matplotlib examples:
- https://matplotlib.org/1.3.1/examples/
- https://github.com/jbmouret/matplotlib_for_papers
- https://www.programcreek.com/python/example/102352/
- matplotlib.pyplot.ticklable_format
- # to list available style format
- import matplotlib.pyplot as plt
- plt.style.available
- style sheets reference:
- https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html
- # customizing matplotlib with style sheets and rcParams
- # Ref: https://matplotlib.org/stable/tutorials/introductory/customizing.html
- # to display the currently active matplotlibrc file was loaded from
- >>> import matplotlib
- >>> matplotlib.matplotlib_fname()
- '/home/user/.venv/dsci/lib/python3.12/site-packages/matplotlib/mpl-data/matplotlibrc'
- The path /user/.../mpl-data/ is where we would like to go and locate the style sheets:
- $ cd .venv/dsci/lib/python3.12/site-packages/matplotlib/mpl-data/
- $ ls
- fonts/ images/ kpsewhich.lua matplotlibrc plot_directive/ sample_data/ stylelib/
- The directory of interest is stylelib/
- $ cd stylelib/
- $ ls
- bmh.mplstyle
- classic.mplstyle
- dark_background.mplstyle
- fast.mplstyle
- fivethirtyeight.mplstyle
- ggplot.mplstyle
- ...
- If we have write privilege to the above mentioned path for stylelib/, we can put
- the custom style sheet into the same directory and invoke the style sheet with:
- >>> plt.style.use("signature")
- If we don't have the write privilege, the only extra thing we would need to do
- is to include the full path of the custom style sheet:
- >>> plt.style.use("/home/user/signature.mplstyle")
- # invoke custom style sheet
- plt.style.use("signature") # apply globally
- with plt.style.context("signature"): # apply locally with context manager
- plt.plot([1, 2, 3, 4])
- # return to default styling
- >>> import matplotlib as mpl
- >>> mpl.rcParams.update(mpl.rcParamsDefault) # reset via rcParams.update
- >>> import matplotlib.pyplot as plt
- >>> plt.style.use("default") # reset with default style sheet
- # matplotlib animation writers list
- >>> import matplotlib.animation as animation
- >>> animation.writers.list()
- ['pillow', 'ffmpeg', 'ffmpeg_file', 'imagemagick', 'imagemagick_file', 'html']
- Python - https://www.python.org/
- Python Course - https://www.python-course.eu/index.php
- Scipy Cookbook
- https://scipy-cookbook.readthedocs.io/items/FrequencySweptDemo.html
- https://book.pythontips.com/en/latest/index.html
- # Some examples
- x = [1,3,5,7,9]
- sum_squared = 0
- for i in range(len(x)):
- sum_squared+=x[i]**2
- for y in x:
- sum_squared+=y**2
- x = [1,3,5,7,9]
- sum_squared = sum([y**2 for y in x]) # pythonic way
- x = [1,2,3,4,5,6,7,8,9]
- even_squared = [y**2 for y in x if y%2==0]
- squared_cubed = [y**2 if y%2==0 else y**3 for y in x]
- # Dictionary comprehension
- x = [1,2,3,4,5,6,7,8,9]
- {k:k**2 for k in x}
- {k:k**2 for k in x if x%2==0}
- {k:k**2 if k%2==0 else k**3 for k in x}
- # Pandas
- import pandas as pd
- # reading data
- data = pd.read_csv('file.csv')
- data = pd.read_csv('file.csv', sep=';', encoding='latin-1', nrows=1000, skiprows=[2,5])
- # (read_csv, read_excel, read_clipboard, read_sql)
- # writing data
- data.to_csv('file.csv', index=None)
- # index=None will simply write the data as it is. If you don't write index=None, you'll
- get an additional first column of 1,2,3,... until the last row.
- # (.to_excel, .to_json, .to_pickle, .to_clipboard)
- # checking the data
- data.shape # gives (#rows, #columns)
- data.describe() # Compute basic statistics
- # seeing the data
- data.head(3)
- # Print the first 3 rows of the data. Similarly to .head(), .tail() will look at the
- last rows of the data.
- data.loc[8] # Print the 8th row
- data.loc[8, 'column_1'] # Print the value of the 8th row on 'column_1'
- data.loc[range(4,6)] # Subset from row 4 to 6(excluded)
- # total sum per column and per row and saving it to Total
- df.loc['Total',:] = df.sum(axis=0) # Total sum per column
- df.loc[:,'Total'] = df.sum(axis=1) # Total sum per row
- To be continue...
- Source: https://towardsdatascience.com/be-a-more-efficient-data-scientist-today-master
- -pandas-with-this-guide-ea362d27386
- # 23 Pandas codes for Data Science
- 1. read in a CSV dataset
- pd.DataFrame.from_csv("csv_file") or pd.read_csv("csv_file")
- 2. read in an excel dataset
- pd.read_excel("excel_file")
- 3. write your data frame directly to csv
- df.to_csv("data.csv", sep=",", index=False)
- 4. basic dataset feature info
- df.info()
- 5. basic dataset statistics
- print(df.describe())
- 6. print data frame in a table
- print(tabulate(print_table, headers=headers))
- 7. list the column names
- df.columns
- 8. drop missing data
- df.dropna(axis=0, how='any')
- 9. replace missing data
- df.replace(to_replace=None, value=None)
- 10. check for NANs
- pd.isnull(object)
- 11. drop a feature
- df.drop('feature_variable_name', axis=1) # axis is either 0 for rows, 1 for columns
- 12. convert object type to float
- pd.to_numeric(df["feature_name"], errors='coerce')
- 13. convert data frame to numpy array
- df.as_matrix()
- 14. get first "n" rows of a data frame
- df.head(n)
- 15. get data by feature name
- df.loc[feature_name]
- 16. apply a function to a data frame
- df["height"].apply(lambda height: 2 * height)
- or
- def multiply(x):
- return x*2
- df["height"].apply(multiply)
- 17. renaming a column
- df.rename(columns = {df.columns[2]: 'size'), inplace=True)
- 18. get the unique entries of a column
- df["name"].unique()
- 19. accessing sub-data frames
- new_df = df[["name", "size"]]
- 20. summary information about your data
- df.sum() # sum of values in a data frame
- df.min() # lowest value of a data frame
- df.max() # highest value
- df.idxmin() # index of the lowest value
- df.idxmax() # index of the highest value
- df.describe() # statistical summary of the data frame, with quartiles, median, etc.
- df.mean() # average values
- df.median() # median values
- df.corr() # correlation between columns
- df["size"].median() # to get these values for only one column
- 21. sorting your data
- df.sort_values(ascending=False)
- 22. boolean indexing
- df[df["size"] == 5]
- 23. selecting values
- df.loc([0], ['size'])
- t = [0.00 0.31 0.59 .90 1.21 1.48 1.81]
- dt = mean(diff(t))
- st = std(diff(t)) # standard deviation of diff(t)
- F = 1/dt
- time = np.linspace(0, 10, 5) for n in range(5):
- for n in range(len(time)): time = np.linspace(0, 10, 5)
- print(n, time) print(n, time)
- a = 1 + 2 + 3 + \ a = (1 + 2 + 3 +
- 4 + 5 + 6 + \ 4 + 5 + 6 +
- 7 + 8 + 9 7 + 8 + 9)
- # 10 Python tips and tricks for writing better code
- 1. Ternary Operatior
- In computer science, a ternary operator is an operator that takes three arguments (or
- operands). The arguments and result can be of different types.
- condition = True condition = True
- if condition: x = 1 if condition else 0 # ternary condition
- x = 1
- else: print(x)
- x = 0
- print(x)
- 2. working with large numbers
- num1 = 10000000000 num1 = 10_000_000_000
- num2 = 100000000 num2 = 100_000_000
- total = num1 + num2 total = num1 + num2
- print(total) print(f'{total:,}')
- 3. context manager
- f = open('test.txt', 'r') with open('test.txt', 'r') as f:
- file_contents = f.read() file_contents = f.read()
- f.close()
- words = file_contents.split(' ') words = file_contents.split(' ')
- word_count = len(words) word_count = len(words)
- print(word_count) print(word_count)
- # Opening two files at the same time with one 'with' statement
- with open("file1", "r") as source, open("file2", "w") as destination:
- destination.write(source.read())
- 4. enumerate function - return both index and value
- names = ['Corey', 'Chris', 'Dave', 'Travis'] ""
- index = 0
- for name in names:
- print(index, name)
- index += 1
- names = ['Corey', 'Chris', 'Dave', 'Travis']
- for index, name in enumerate(names): # default count from 0
- for index, name in enumerate(names, start=1):
- print(index, name)
- 5. zip function (unpacking) - loop over two or more lists at once
- names = ['Peter Parker', 'Clark Kent', 'Wade Wilson', 'Bruce Wayne']
- heroes = ['Spiderman', 'Superman', 'Deadpool', 'Batman']
- for index, name in enumerate(names):
- hero = heroes[index]
- print(f'{name} is actually {hero}')
- names = ['Peter Parker', 'Clark Kent', 'Wade Wilson', 'Bruce Wayne']
- heroes = ['Spiderman', 'Superman', 'Deadpoolv, 'Batman']
- for name, hero in zip(names, heroes):
- print(f'{name} is actually {hero}')
- names = ['Peter Parker', 'Clark Kent', 'Wade Wilson', 'Bruce Wayne']
- heroes = ['Spiderman', 'Superman', 'Deadpool', 'Batman']
- universes = ['Marvel', 'DC', 'Marvel', 'DC']
- for name, hero, universe in zip(names, heroes, universes):
- print(f'{name} is actually {hero} from {universe}')
- names = ['Peter Parker', 'Clark Kent', 'Wade Wilson', 'Bruce Wayne']
- heroes = ['Spiderman', 'Superman', 'Deadpool', 'Batman']
- universes = ['Marvel', 'DC', 'Marvel', 'DC']
- # to print tuples of all three values
- for value in zip(names, heroes, universes):
- print(value)
- 6. To be continue ...
- # Pylint
- pylint program.py
- # Valgrind in Python
- valgrind python3 file.py # to use valgrind
- valgrind --tool=massif python3 file.py # use of memory during program execution
- # tabs into space (to fix python indentation)
- pip install autopep8
- autopep8 script.py # print only
- autopep8 -i script.py # write file
- On most UNIX-like systems, can also run:
- expand -t4 oldfile.py > newfile.py
- from the command line, changing the number if you want to repalce tabs with a
- number of spaces other than 4. You can easily write a shell script to do this
- with a bunch of files at once, retaining the original file names.
- # profiling in python
- python3 -m cProfile -o file.prof file.py
- Using the -o switch will output the profiler results to the file.prof
- # line_profiler and kernprof (Ref: https://github.com/pyutils/line_profiler)
- pip install line_profiler
- kernprof will create an instance of LineProfiler and insert it into the
- __builtins__ namespace with the name profile. It has been written to be used
- as a decoratro, so in your script, you decorate the functions you want to
- profile with @profile.
- @profile
- def function(x):
- ...
- kernprof -l program.py
- mpiexec -np 2 kernprof -l program.py
- the default behavior of kernprof is to put the results into a binary file
- program.py.lprof you can tell kernprof to immediately view the formatted
- results at the terminal with the [-v/--view] option. Otherwise, you can
- view the results later like so:
- $ python -m line_profiler program.py.lprof
- kernprof -lv program.py
- mpiexec -np 2 kernprof -lv program.py
- kernprof also works with cProfile, its third-party incarnation lsprof, or the
- pure-Python profile module depending on what is available.
- # memory profiler (Ref: https://github.com/pythonprofilers/memory_profiler)
- pip install memory_profiler
- use mprof to generate a full memory usage report of your executable and to plot it:
- mprof run program.py
- mprof plot
- line-by-line memory usage:
- @profile
- def function():
- ...
- python -m memory_profiler program.py
- a function decorator is also available:
- from memory_profiler import profile
- @profile
- def function():
- ...
- in this case the script can be run without specifying -m memeory_profiler in the
- command line.
- in function decorator, you can specify the precision as an argument to the decorator
- function:
- from memory_profiler import profile
- @profile(precision=4)
- def function():
- ...
- if a python script with decorator @profile is called using -m memory_profiler in the
- command line, the precision parameter is ignored.
- # complex number
- >>> complex(2,3)
- (2+3j)
- >>> z = complex(input('Enter a complex number: '))
- >>> z = 2+3j
- >>> z.real
- 2
- >>> z.imag
- 3
- >> z.conjugate()
- (2-3j)
- # fractional number
- >>> from fractions import Fraction
- >>> Fraction(2, 3)
- 2/3
- >>> Fraction(2/3)
- 2/3
- >>> f = Fraction(input('Enter a fractional number: '))
- >>> type(f)
- >>> int('4.8')
- Error
- >>> int(float('4.8'))
- 4
- >>> N = 4.0
- >>> N.is_integer()
- True
- >>> N = 4.8
- >>> N.is_integer()
- False
- # random number generation
- np.random.randint(2, size=10)
- array([1, 0, 0, 0, 1, 1, 0, 0, 1, 0])
- np.random.randint(1, size=10)
- array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
- # generate a 2 x 4 array of ints between 0 and 4, inclusive:
- np.random.randint(5, size=(2, 4))
- array([[4, 0, 2, 1],
- [3, 2, 2, 0]])
- Ref: https://docs.scipy.org/doc/numpy-1.15.1/reference/generated/numpy.random.randint.html
- # Array
- a = np.array([11, 12, 13, 14])
- b = np.array([1, 2, 3, 4])
- c = a - b # array subtraction
- b**2 # squating an array
- np.cos(b) # a trigonometric function performed on the array
- b < 2 # conditional operations
- # help
- import numpy
- help(numpy)
- # use 'enumerate' function in loops instead of creating an 'index' variable
- Harmful:
- my_container = ['Larry', 'Moe', 'Curly']
- index = 0
- for element in my_container:
- print('{} {}'.format(index, element))
- index += 1
- Idiomatic:
- my_container = ['Larry', 'Moe', 'Curly']
- for index, element in enumerate(my_container):
- print('{} {}'.format(index, element))
- # Tips and Tricks in Python
- 1. Swapping of Two numbers
- x, y = 10, 20
- print(x, y)
- Result: 10 20
- x, y = y, x
- print(x, y)
- Result: 20 10
- 2. Reversing a string in Python
- a = "GeeksForGeeks"
- print("Reverse is", a[::-1])
- Result: Reverse is skeeGroFskeeG
- 3. Create a single string from all the elements in list
- a = ["Geeks", "For", "Geeks"]
- print(" ".join(a))
- Result: Geeks For Geeks
- 4. Chaining of Comparison Operatorst
- n = 10
- result = 1 < n n <= 9
- print(result)
- Result: True
- False
- 5. Print the file path of imported modules
- import os;
- import socket;
- print(os)
- print(socket)
- 6. Use of Enums
- class MyName:
- Geeks, For, Geeks = range(3)
- print(MyName.Geeks)
- print(MyName.For)
- print(MyName.Geeks)
- Result: 2
- 1
- 2
- 7. Return multiple values from functions
- def x():
- return 1, 2, 3, 4
- a, b, c, d = x()
- print(a, b, c, d)
- Result: 1 2 3 4
- 8. Find the most frequent value in a list
- test = [1, 2, 3, 4, 2, 2, 3, 1, 4, 4, 4]
- print(max(set(test), key = test.count))
- Result: 4
- # Lists
- empty_list = list()
- also_empty_list = []
- zeros_list = [0] * 5
- print(zeros_list]
- >>> [0, 0, 0, 0, 0]
- empty_list.append(1)
- print(empty_list)
- >>> [1]
- print(len(empty_list))
- >>> 1
- # List indexing
- list_var = range(10)
- print(list_var)
- >>> [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
- print(list_var[4])
- >>> 4
- print(list_var[4:7])
- >>> [4, 5, 6]
- print(list_var[0::3]) # empty index means to the beginning/end
- >>> [0, 3, 6, 9]
- print(list_var)
- >>> [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
- print(list_var[-1])
- >>> 9
- print(list_var[::-1])
- >>> [9, 8, 7, 6, 5, 4, 3, 2, 1, 0]
- print(list_var[3:1:-1])
- >>> [3, 2]
- # Dictionaries
- empty_dict = ()
- also_empty_dict = {}
- filled_dict = (3: 'Hello, ', 4: 'world!')
- print(filled_dict[3] + filled_dict[4])
- >>> Hello, World!
- filled_dict[5] = 'New String'
- print(filled_dict)
- >>> (3: 'Hello, ', 4: 'World!', 5: 'New String')
- del filled_dict[3]
- print(filled_dict)
- >>> (4: 'World!', 5: 'New String')
- print(len(filled_dict))
- >>> 2
- # Functions, Lambda functions
- def add_numbers(a, b):
- return a + b
- print(add_numbers(3, 4))
- >>> 7
- lambda_add_numbers = lambda a, b: a + b
- print(lambda_add_numbers(3, 4))
- >>> 7
- # Loops, List and Dictionary Comprehensions
- for i in range(10):
- print('Looping %d' %i)
- >>> looping 0
- ...
- >>> looping 9
- filled_list = [a/2 for a in range(10)]
- print(filled_list)
- >>> [0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5]
- filled_dict = (a:a**2 for a in range(5)]
- print(filled_dict)
- >>> (0: 0, 1: 1, 2: 4, 3: 9, 4: 16)
- # zip
- L1 = [1,2,3,4]
- L2 = [5,6,7,8]
- In: list(zip(L1,L2))
- Out: [(1,5), (2,6), (3,7), (4,8)]
- In: for (x,y) in zip(L1,L2):
- print(x, y, '--', x+y)
- Out: 1 5 -- 6
- ........
- 4 8 -- 12
- # Python Lambda Functions
- def average(x, y):
- return (x+y)/2
- avg = average(2, 5)
- average = lambda x, y: (x+y)/2
- var = [1, 5, -2, 3, -7, 4]
- sorted_var = sorted(var)
- Ref: https://www.pythonforthelab.com/blog/intro-to-python-lambda-functions/
- # Beyond the for-loop
- integers = range(0, 10)
- even = []
- for i in integers:
- if i%2 == 0:
- even.append(i)
- >>> even
- >>> [0, 2, 4, 6, 8]
- integers = range(0, 10)
- even = filter(lambda x: x%2 == 0, integers)
- integers = range(0, 10)
- def is_even(x):
- return x%2 == 0
- even = filter(is_even, integers)
- integers = range(0, 10)
- even = [x for x in integers if x%2 == 0]
- # Map
- integers = range(0, 10)
- list(map(lambda x: x*x, integers))
- >>> [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
- integers = range(0, 10)
- [x*x for x in integers]
- >>> [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
- # Reduce
- from functools import reduce
- integers = range(1, 10)
- reduce(lambda x, y: x*y, integers)
- >>> 362880
- # Sum
- integers = range(1, 10)
- sum(integers)
- >>> 45
- # Any and All
- any([False, True, False])
- >>>True
- all([False, True, False])
- >>> False
- To check for even numbers in a list:
- integers = range(1, 10)
- any(x%2 == 0 for x in integers)
- >>> True
- all(x%2 == 0 for x in integers)
- >>> False
- # List Methods
- list.append(x)
- list.extend(iter)
- list.insert(i, x)
- list.remove(x)
- list.clear()
- list.copy()
- list.reverse()
- list.pop([i])
- list.sort()
- # format text
- greeting = 'Hello'
- name = 'Raman'
- message = '{}, {}. Welcome!'.format(greeting, name)
- message = f'{greeting), {name}. Welcome!'
- print(message)
- $ Hello, Raman. Welcome!
- message = f'{greeting}, {name.upper()}. Welcome!'
- print(message)
- $ Hello, RAMAN. Welcome!
- print(dir(name)) # dir function
- print(help(str)) # help function
- print(help(str.lower)) # help function
- Creating a vector:
- vector_row = np.array([1, 2, 3])
- vector_col = np.array([[1], [2]. [3]])
- Creating a matrix:
- matrix = np.array([[1,2], [1,2], [1,2]])
- matrix_obj = np.mat([[1,2], [1,2], [1,2]])
- # Seaborn
- import seaborn as sns
- splot = sns.regplot(x='fieldname1', y='fieldname2', data=dfname, fit reg=False)
- splot.set(xscale='log') # logrithm plot
- Note: https://cmdlinetips.com/2019/04/how-to-make-scatter-plot-in-python/
- # convert an array into a list
- np.array([1,2,3], [4,5,6]).tolist()
- [[1,2,3], [4,5,6]]
- # num2words
- from num2words import num2words
- print(num2words(30)) # thirty-six
- print(num2words(30, to = 'ordinal') # thirty-six
- print(num2words(30, to = 'ordinal_num')) # thirty-sixth
- print(num2words(30, to = 'year')) # 36th
- print(num2words(30, to = 'currency')) # zero euro, thirty-six cents
- print(num2words(30, lang = 'es')) # treinta y seis
- # to find out which file you're importing
- import math
- print(math.__file__)
- /usr/local/lib/python3.12/lib-dynload/math.cpython-312-x86_64-linux-gnu.so
- import numpy
- print(numpy.__file__)
- /home/saran/.venv/fdtd/lib/python3.12/site-packages/numpy/__init__.py
- # Checking which indices have null for column c
- pd.isnull(df['c'])
- # Checking which indices don't have null for column c
- pd.notnull(df['c'])
- # Selecting rows of df where c is not null
- df[pd.notnull(df['c'])]
- # Selecting rows of df where c is null
- df[pd.isnull(df['c'])]
- # Selecting rows of column c of df where c is not null
- df['c'][pd.notnull(df['c'])]
- What to do about outliers?
- 1. Remove the case. If you have many cases and there does not appear to be an
- explanation is that it is in error, you can simply get rid of it.
- 2. Assign the next value nearer to the median in place of the outlier value.
- You will find that this approach leaves the distribution close to what it would
- be without the value. You can use this approach if you have few cases and are
- trying to maintain the number of observations you do have.
- 3. Calculate the mean of the remaining values without the outlier and assign
- that to the outlier case. While I have seen this frequently, I don't really
- understand its justification and I think it distorts the distribution more than
- the previous solution.
- EDA is one of the most crucial aspects in any data science projects, and an
- absolutely must have before commencement of any machine learning projects.
- Achieving a high degree of certainty and accuracy on the validity,
- interpretation and applicability of the data set and the project in general
- ensures desired outcomes.
- # box-plot
- We analyse Uni-variate outlier i.e. we used DIS column only to check the
- outlier. But we can do multivariate outlier analysis too. Can we do the
- multivariate analysis with box-plot ?
- Well it depends, if you have a categorical values then you can use that with
- any continuous variable and do multivariate outlier analysis.
- # Visulization:
- Matplotlib - https://matplotlib.org
- Seaborn - https://seaborn.pydata.org
- Plotly - https://plot.ly
- Bokeh - https://bokeh.pydata.org
- Pygal - http://pygal.org/en
- Dash - https://plot.ly/products/dash
- Altair - https://altair-viz.github.io
- http://www.intellspot.com/python-visualization-tools/
- Python Programming Books:
- 01. Automate the Boring Stuff with Python by Al Sweigart
- 02. Effective Python: 59 Specific ways to write better Python by Brett Slatkin
- 03. Fluent Python: Clear, Concise and Effective Programming by Luciano Ramalho
- 04. Hello Web App by Tracy Osborn
- 05. Invent Your Own Computer Games with Python by Al Sweigart
- 06. Learning Python by Mark Lutz and David Ascher
- 07. Learning Python: Learn to code like a professional with Python by Fabrizio Romano
- 08. Learn to Program with Python 3 by Irv Kalb
- 09. Programming Arcade Games with Python and Pygame by Paul Craven
- 10. Python 101 by Mike Driscoll
- 11. Python3 Object-oriented Programming by Dusty Philips
- 12. Python Cookbook by David Beazley and Brian K. Jones
- 13. Python Scripting with Scribus by Greg Pittman
- 14. Python Tricks: The Book by Dan Bader
- 15. Scaling Python by Julien Danjou
- 16. The Hacker's Guide to Python by Julien Danjou
- 17. The Quick Python Book by Naomi Ceder
- 18. Treading on Python: Volume 2 Intermediate Python by Matt Harrison
- """
- Error: After October 2020 you may experience errors when installing or updating packages.
- This is because pip will change the way that it resolves dependency conflicts.
- We recommend you use --use-feature=2020-resolver to test your packages with the new
- resolver before it becomes the default.
- """
- # use r code in python
- from rpy2.robjects import pandas2ri
- pandas2ri.activate()
- from rpy2.robjects import r
- r.data('iris')
- df = pandas2ri.rpy2py(r[name])
- df.to_csv("/path/to/file.csv", sep=',')
- df.to_csv("/path/to/file.csv", sep=',', encoding='utf-8')
- # save session in IPython
- 1. %logstart # saves IPython session
- 2. %save session.py 1-N
- 3. %%file session.py
- 4. import readline
- readline.write_history_file("/home/saran/session.py")
- 5. %store session.py
- 6. %store -r session.py
- # linux commands in IPython
- ls
- cd
- pwd
- %run script.py
- %load script.py
- # jupyter qtconsole
- jupyter qtconsole --style monokai
- jypyter qtconsole -h
- # jypyter notebook remote access
- jupyter notebook --ip=0.0.0.0 --port=8080 --no-browser
- First make sure you install Jupyter notebook in both remote and local host.
- In remote host, open the terminal, change directory to where you have your notebooks and type:
- jupyter notebook --no-browser --port=8889
- In your local computer, open terminal and then type
- ssh -N -f -L localhost:8888:localhost:8889 user@remote_hostname
- Now open web browser and type:
- localhost:8888
- jupyter --paths
- jupyter kernelspec list
- # empty line in markdown
-
- # using find command
- find . -type f -name iris.csv
- # UnicodeDecodeError when reading csv file in pandas with python
- import pandas as pd
- df = pd.read_csv('/path/to/file/file_name.csv', engine='python')
- alternate solution:
- - open the csv file in vi editor
- - save the file in utf-8 format
- then,
- import pandas as pd
- df = pd.read_csv('/path/to/file/file_name.csv', encoding='utf-8')
- other encodings include, 'cp1252', 'ISO-8859-1'
- # unpacking data
- a, b, c = 1, 2, 3 # print(a, b, c) = 1 2 3
- a, b, c = [1, 2, 3] # print(a, b, c) = 1 2 3
- # dictionary
- mydict = {'a': 1, 'b': 2. 'c': 3}
- mydict.keys() # keys
- mydict.values() # values
- for n in mydict.keys():
- print(mydict[n])
- 1
- 2
- 3
- # convert python list to numpy array
- import numpy as np
- mylist = [1, 2, 3, 4, 5]
- myarry = np.array(mylist)
- print(myarry)
- print(myarry.shape)
- mylist = [[1, 2, 3], [4, 5, 6]]
- myarry = np.array(mylist)
- print(myarry)
- print(myarry.shape)
- print('First row:', myarry[0])
- print('Last row:', myarry[-1])
- print('Specific row and column:', myarry[0,2])
- print('All last column values:', myarry[:,-1])
- # number of dimensions
- myarry.ndim
- # size of an array
- myarry.size
- # shape of an array
- myarry.shape
- # set option in pandas
- from pandas import set_option
- set_option('display.width', 100)
- set_option('precision', 3)
- df.describe()
- # plot groupby (for classification problems)
- df.groupby('colname').size()
- plt.plot(df.groupby('colname').size())
- plt.show()
- # groupby size, count, describe
- df.groupby('colname').size()
- df.groupby('colname').count()
- df.groupby('colname').describe()
- # correlations between attributes
- Some machine learning algorithms like linear and logistic regression can suffer poor
- performance if there are highly correlated attributes in your dataset.
- df.corr()
- df.corr(method = 'pearson') # assumes a normal distribution of the attributes involved
- # skewness - degree of distortion
- A symmetrical distribution has skewness value = 0 (gaussion/normal distribution)
- fairly symmetrical: -0.5 to +0.5
- moderately skewed: -1 to -0.5 or +0.5 to +1
- highly skewed: less than -1 or greater than +1
- df.skew() # calculate skewness of a pandas dataframe
- df.hist() # histogram plot for skewness
- plt.show()
- #
- The code:
- np.random.seed(1)
- np.random.normal(loc = 0, scale = 1, size = (3,3))
- Operates effectively the same as this code:
- np.random.seed(1)
- np.random.randn(3,3)
- # number of elements
- row, col = df.shape
- print(row, col)
- print(df.shape[0], df.shape[1])
- # Python Notes [Higher Level Programming]
- Books and tutorials:
- Python Library Reference
- Python 3 tutorial
- Think Python
- # Build-in documentation
- pydoc module
- pydoc module.func
- Example: !pydoc sys.exit
- # Running the script from the command-line
- #!/usr/bin/env python3 # kind of script language interpreter to use
- from math import sin # access library functionality like the function sin
- import sys # and the list sys.arg (of command-line arguments)
- # read first command-line argument and convert it to a floating point object
- x = float(sys.argv[1])
- # print out the result using a format string
- print("sin({0}) = {1}".format(x, sin(x)))
- # complete control of the formating of floats (similar to the C's printf syntax)
- print("sin({x:q}) = {s:.3f}".format(x=x, s=sin(x)))
- $ python program.py 0.8 # python - name of the interpreter
- sin(0.8) = 0.7173560908995228
- $ chmod u+x program.py # make the file executable
- $ ./program.py 0.8
- sin(0.8) = 0.7173560908995228
- # Python as a calculator
- In[]: 1+2
- 3
- In[]: 4.5/3 + (1+2)*3
- 10.5
- in[]: 4**5
- 1024
- In[]: a = 1+2j
- In[]: b = 3-5j
- In[]: a*b
- (13+1j)
- In[]: from math import log10
- log10(5)
- 0.6989700043360189
- # variables and data types
- 'some string' is equivalent to "some string"
- text = """ large portions of a text can be conveniently placed inside
- triple-quoted strings (newlines are preserved)"""
- # special characters in strings
- use the backslash \ to escape special characters
- s = "\"This is a quote\" and \n here comes a backslash: \\"
- print(s)
- "This is a quote" and
- here comes a backslash: \
- # string concatenation
- "condensed "*3 + "matter"
- condensed condensed condensed matter
- quote = "I will not eat chips all day"
- (quote + ", ") * 10 + quote
- # slicing notation [start:end]
- quote[2:6] # will
- quote[:6] # I will
- quote[7:] # not eat chips all day
- quote[2:-4] # will not eat chips all
- python strings are immutable, meaning that they cannot be changed
- quote[1] = "x"
- TypeError: 'str' object does not support item assignment
- if one wants to change a string, one needs to create a new one:
- quote = quote[:1] + "x" + quote[2:]
- print(quote)
- Ixwill not eat chips all day
- >>> 'day' in quote
- True
- >>> quote.find('i')
- 3
- >>> quote.split()
- ['I', 'will', 'not', 'eat', 'chips', 'all', 'day']
- >>> quote.replace('chips', 'salad')
- 'I will not eat salad all day'
- >>> quote.lower()
- 'i will not eat chips all day'
- >>> quote.upper()
- 'I WILL NOT EAT CHIPS ALL DAY'
- >>> quote.strip() # remove leading/trailing blanks
- 'I will not eat chips all day'
- # lists
- lists can contain items of different type, though in practice they often
- have the same type
- mylist = ['institute', 'of', 'mathematical', 'sciences']
- mylist = ['institute', 4, True]
- # list operations
- >>> mylist[0] # indexing
- 'institute'
- >>> mylist[1:] # slicing
- [4, True]
- >>> newlist = mylist + ["!"]*3
- >>> newlist
- ['institute', 4, 'True', '!', '!', '!']
- in constrast to strings, lists are mutable and can be changed
- mylist = [11, 12, 14]
- mylist[2] = 13
- mylist
- [11, 12, 13]
- we can also append additional items to a list
- mylist.append(14)
- mylist
- [11, 12, 13, 14]
- # tuples
- tuples are very similar to lists, but they are immutable, just like strings
- mytuple = ('a string', 2.5, 6, 'another string')
- mytuple = 'a string', 2.5, 6, 'another string' # shorter notation
- mytuple[1] = -10 # Error, tuple cannot be changed
- instead we need to create a new tuple with the changed values, for example
- by converting the tuple to list, changing it, and converting it back to a
- tuple:
- l = list(mytuple)
- l[1:3] = ["is", "not"]
- mytuple = tuple(l)
- mytuple
- ('a string', 'is', 'not', 'another string')
- # change backends in matplotlib
- matplotlib.get_backend() # find backend
- matplotlib.use('TKAgg', warn=False, force=True) # use TKAgg
- import matplotlib.pyplot as plt
- plt.switch_backend('TKAgg')
- # what backends are available and where
- import matplotlib as m; help(m);
- import matplotlib as m
- print('I: {}\nN: {}'.format(m.rcsetup.interactive_bk, m.rcsetup.non_interactive_bk))
- import matplotlib as m
- p = m.get_backend(); print("current backend is:", p)
- import matplotlib as m
- p = m.matplotlib_fname(); print("The matplotlibrc is located at:\n", p)
- # setting the back-end
- There are 3 ways to configure backend:
- 1. setting the rcParams["backend"] (default: 'Agg') parameter in matplotlibrc file
- 2. setting the MPLBACKEND environment (shell) variable
- 3. using the function matplotlib.use()
- # using the backend
- import matplotlib
- matplotlib.use('TKAgg', force=True) # Agg rendering to a TK canvas
- matplotlib.use('wxcairo', force=True) # Cairo rendering to a wxwidgets canvas
- matplotlib.use('wxagg', force=True) # Agg rendering to a wxwidgets canvas
- matplotlib.use('webagg', force=True) # On show() will start a tornado server with an interactive figure
- matplotlib.use('qt5cairo', force=True) # Cairo rendering to a Qt5 canvas
- matplotlib.use('qt5agg', force=True) # Agg rendering to a Qt5 canvas
- # extra info
- pip install pycairo # Cairo: GTK3 based backend (replaces: cairocffi)
- pip install mplcairo # Cairo: Easy & Specific for matplotlib
- pip install PyQt5 # Qt5: Require: Qt's qmake tool
- pip install Pyside2 # Qt5: Require: shiboken2 & clang lib bindings
- pip install wxPython # wxAgg:
- pip install tornado # webAgg: Require: pycurl, twisted, pycares
- # change the backend
- 1. First locate the matplotlibrc file:
- import matplotlib
- matplotlib.matplotlib_fname()
- 2. Open terminal and do:
- cd /users/serafeim/.matplotlib/
- ls
- 3. Edit the file (if it does not exist use this command: touch matplotlib to create it):
- vi matplotlibrc
- 4. Save in matplotlibrc:
- backend: TKAgg
- cd .envn/dsci/lib/python3.7/site-packages/matplotlib/mpl-data/
- vi matplotlibrc
- line No. 81
- backend: TKAgg
- Both Agg and TkAgg do not require any dependencies beyond Python's standard library. If
- need to save to files and not plt.show(), using Agg instead (just replace it where TkAgg
- appears below).
- Either add the following line to ~/.config/matplotlib/matplotlibrc:
- backend: TkAgg
- or the following lines to python file:
- import matplotlib
- matplotlib.use('TkAgg')
- import matplotlib.pyplot as plt
- To display where the currently active matplotlibrc file was loaded from, on can do the
- following:
- >>> import matplotlib
- >>> matplotlib.matplotlib_fname()
- '/home/user/.config/matplotlib/matplotlibrc'
- Ref: https://matplotlib.org/stable/tutorials/introductory/customizing.html
- # jupyter notebook with inline matplotlib
- jupyter notebook --matplotlib=inline
- # garbage collection
- >>> x = 9
- >>> print(9)
- 9
- >>> del x
- >>> print(x)
- NameError: name 'x' is not defined
- # tuple:
- A tuple is a general way of grouping together a number of values with a variety of types
- into one compound type. Tuples have a fixed length: once declared they vannot grow or
- shrink in size.
- # functions:
- Functions start with a header introduced by the def keyword. The indented block of code
- following the : is run when the function is called. return is another keyword uniquely
- associated with functions. When Python encounters a return statement, it exits the
- function immediately, and passes the value on the right hand side to the calling context.
- def least_difference(a, b, c):
- """ Return the smallest difference between any two numbers among a, b, c.
- >>> least_differnce(1, 5, -5)
- 4 """
- diff1 = abs(a - b)
- diff2 = abs(b - c)
- diff3 = abs(a - c)
- return min(diff1, diff2, diff3)
- print(
- least_difference(1, 10, 100),
- least_difference(1, 10, 10),
- least_difference(5, 6, 7)
- )
- help(least_difference) # display the docstring
- The convention of including 1 or more example calls in a function's docstring is far from
- universally observed, but it can be very effective at helping someone understand your
- function.
- print(1, 2, 3, sep=' < ')
- 1 < 2 < 3
- # functions applied to functions
- def multiply_by_five(x):
- return 5 * x
- def call(fn, arg):
- """ Call fn on arg """
- return fn(arg)
- def squared_call(fn, arg):
- """ Call fn on the result of calling fn on arg """
- return fn(fn(arg))
- print(
- call(multiply_by_five, 1),
- squared_call(multiply_by_five, 1),
- sep='\n'
- }
- 5
- 25
- Functions that operate on other functions are called "higher-order functions".
- By default, max returns the largest of its arguments. But if we pass in a function using
- the optional 'key' argument, it returns the argument x that maximizes key(x) (aka the
- 'argmax').
- def modulus_5(x):
- """ Returns the remainder of x after dividing by 5 """
- return x % 5
- print(
- max(100, 51, 14), # print biggest number
- max(100, 51, 14, key=modulus_5), # print biggest modulo 5 number
- sep='\n'
- )
- 100
- 14
- # find the word-size, int_info, float_info
- import sys
- sys.maxsize
- sys.int_info
- sys.float_info
- # NotADirectoryError: [Errno 20] Not a directory: 'dvipng'
- sudo apt-get install dvipng
- # IPython and the pylab mode
- $ ipython --pylab
- # built-in factorial
- import math
- math.factorial(1000)
- # user defined factorial
- def factorial(n):
- """returns n!"""
- if type(n) != type(0):
- raise TypeError, "integer required as input"
- return 1 if n < 2 else n * factorial(n-1)
- factorial(998)
- # cpu count
- import os
- os.cpu_count()
- # hidden gems in numpy
- import numpy as np
- np.iinfo(np.int16)
- np.iinfo(np.int32)
- np.iinfo(np.int64)
- x, y = np.ogrid[1:10, 1:5]
- # solve the system of equations x + 2y = 1 and 3x + 5y = 2
- import numpy as np
- A = np.array([[1, 2], [3, 5]])
- b = np.array([1, 2])
- x = np.linalg.solve(A, b)
- print(x) # array([-1., 1.])
- np.allclose(np.dot(A, x), b) # check that the solution is correct -Output: True
- Ref: https://numpy.org/doc/stable/reference/generated/numpy.linalg.solve.html
- # performance check
- $ /usr/bin/time -p ./program.py
- $ /usr/bin/time --verbose ./program.py
- $ python -m cProfile -s cumulative program.py
- $ python -m cProfile -o profile.stats program.py
- $ kernprof -l -v ./program.py
- $ python -m memory_profiler program.py
- $ perf stat -e cycles, instructions, cache-references, cache-misses, branches,\
- branch-misses, task-clock, faults, minor-faults, cs, migrations python program.py
- $ python -m timeit '[x**0.5 for x in range(1000)]'
- $ python -m timeit -s 'from math import sqrt' '[sqrt(x) for x in range(1000)]'
- $ python -m cProfile -s ncalls program.py # which function is called the most
- number of times
- $ python -m cProfile -o program.stats program.py
- $ python -m pstats program.stats
- $ python -m cProfile -s tottime program.py
- sort string meaning
- calls call count
- cumulative cumulative time
- cumtime cumulative time
- file file name
- filename file name
- module file name
- ncalls call count
- pcalls primitive call count
- line line number
- name function name
- nfl name/file/line
- stdname standard name
- time internal time
- tottime internal time
- # find packages that should or should not be in requirements for a project:
- https://pypi.org/project/pip-check-reqs/2.0/
- https://packaging.python.org/en/latest/tutorial/#creating-your-own-project
- # remove a package and its unused dependencies
- Ref:https://pypi.org/project/pip-autoremove/
- pip-autoremove <package> -y
- # binary, octal, hexadecimal
- bin(64)
- oct(64)
- hex(64)
- # mayavi reference
- 1. 3D plotting with Mayavi
- https://scipy-lectures.org/packages/3d_plotting/index.html
- 2. Installation
- https://docs.enthought.com/mayavi/mayavi/installation.html
- https://www.math.univ-paris13.fr/~cuvelier/Python/Python-Mayavi-Qt5-centOS7.html
- 3. Mayavi: 3D scientific data visualization and plotting in Python
- https://mayavi.readthedocs.io/en/latest/
- 4. Example gallery
- https://docs.enthought.com/mayavi/mayavi/auto/examples.html
- 5. 3D plotting with Mayavi
- http://python4esac.github.io/plotting/mayavi_example.html
- 6. SAM's Scientific Python Tools
- https://metaphor.ethz.ch/fsdb/sam/PythonTutorial/tips_mayavi2.html
- 7. Better 3d visualizations with Mayavi
- https://wwwstud.fh-zwickau.de/jef19jdw/teaching/pti01830/mayavi.html
- 8. Mayavi surf
- https://wizardforcel.gitbooks.io/scipy-cookbook-en/content/61.html
- 9. Mayavi github codes - https://github.com/enthought/mayavi
- # monitoring CUDA activity on a GPU
- watch -n 0.1 nvidia-smi
- nvidia-smi -lms 500 (every 500 milliseconds)
- watch nvidia-smi -q g 0 -d UTILIZATION
- watch gpustat -cp
- watch -c gpustat -cp --color
- watch -n 0.5 -c gpustat -cp --color
- Ref: https://stackoverflow.com/questions/8223811/a-top-like-utility-for-monitoring-cuda-activity-on-a-gpu
- # control GPUs with nvidia-smi
- nvidia-smi
- nvidia-smi -L # to list all available NVIDIA devices
- nvidia-smi --query-gpu=index,name,uuid,serial --format=csv # to list certain details about each GPU
- nvidia-smi -q -d SUPPORTED_CLOCKS # listing of available clock speeds for each GPU
- nvidia-smi -q -d CLOCK # to review the current GPU clock speed, default clock speed, and maximum possible clock speed
- nvidia-smi -q -d PERFORMANCE
- nvidia-smi topo --matrix
- nvidia-smi nvlink --status
- nvidia-smi nvlink --capabilities
- nvidia-smi -i 0 -q # list all available data on a particular GPU, specify the ID of the card with -i
- Ref: https://www.microway.com/hpc-tech-tips/nvidia-smi_control-your-gpus/
- # MPI for Python
- https://mpi4py.readthedocs.io/en/stable/index.html
- http://education.molssi.org/parallel-programming/03-distributed-examples-mpi4py/index.html
- https://www.ibm.com/docs/en/smpi/10.4?topic=command-mpirun-options
- mpiexec --help
- mpiexec -n 4 python program.py
- mpiexec --use-hwthread-cpus python program.py
- mpiexec -n 4 --oversubscribe python program.py
- mpiexec --mca plm_rsh_args -x ./program.py
- # MPI communication
- 1. communication for generic python objects
- * use "lower case" methods: send(), receive()
- 2. communication for buffer-provider objects (e.g. numpy arrays)
- * use "upper case" methods: Send(), Receive()
- # importing mpi library
- from mpi4py import MPI
- # getting important information
- comm = MPI.COMM_WORLD
- rank = comm.Get_rank()
- size = comm.Get_size()
- name = MPI.Get_processor_name()
- comm.Barrier()
- # collective communication
- MPI_MAX - returns the maximum element
- MPI_MIN - returns the minimum element
- MPI_SUM - sums the elements
- MPI_PROD - multiplies all elements
- MPI_LAND - performs a logical "and" across the elements
- MPI_LOR - performs a logical "or" across the elements
- MPI_MAXLOC - the maximum value and the rank of the process that owns it
- MPI_MINLOC - the minimum value and the rank of the process that owns it
- # operating system information
- import os
- os.uname()
- # anonymous function
- list1 = [2, 18, 9, 22, 17, 24, 8, 12, 27]
- filter(lambda x: x%3 == 0, list1)
- for n in filter(lambda x: x%3 == 0, list1):
- print(n)
- map(lambda x: x*2 + 10, list1)
- for n in map(lambda x: x*2 + 10, list1):
- print(n)
- reduce(lambda x,y: x + y, list1) # is equivalent to sum(list1)
- # built-in functions (source: http://docs.python.org/2/library/functions.html)
- abs() divmod() input() open() staticmethod()
- all() enumerate() int() ord() str()
- any() eval() isinstance() pow() sum()
- basestring() execfile() issubclass() print() super()
- bin() file() iter() property() tuple()
- bool() filter() len() range() type()
- bytearray() float() list() raw_input() unichr()
- callable() format() locals() reduce() unicode()
- chr() frozenset() long() reload() vars()
- classmethod() getattr() map() repr() xrange()
- cmp() globals() max() reversed() zip()
- compile() hasattr() memoryview() round() __import__()
- complex() hash() min() set() apply()
- delattr() help() next() setattr() buffer()
- dict() hex() object() slice() coerce()
- dir() id() oct() sorted() intern()
- # putting an if-elif-else statement on one line
- # fucntion
- def f(x):
- return 1 if x == 0 else 2*x if x == 1 else 10*x
- def f(x):
- return 1 if x == 0 else(2*x if x == 1 else 10*x)
- def f(x):
- return (x == 0 and 1) or (x == 1 and 2*x) or 10*x
- # code format
- black -> pylint -> pytype -> pytest
- # lint using pylint
- pylint --list-msgs # get list of pylint warnings
- pylint --help-msg=C6409 # get more information on a particular message
- # array size
- x = np.array([0.2, 6.4, 3.0, 1.6])
- for n in range(x.size):
- print(n)
- 0
- 1
- 2
- 3
- n = x.size
- print(n)
- 4
- # show config of the numpy, scipy package
- >>> import numpy as np
- >>> np.show_config()
- # print the installed module location
- $ python -c "import pip; print(pip)"
- $ python -c "import scipy; print(scipy)"
- # QuTiP: Quantum Toolbox in Python
- >>> import qutip
- >>> qutip.about()
- # integer to character
- The built-in function chr() takes an integer argument and produces the
- correspoinding character.
- >>> chr(65)
- 'A'
- >>> chr(66)
- 'B'
- >>> for i in range(65, 75):
- print(chr(i))
- # character to integer
- The built-in function ord() is effectively the inverse of chr(). ord() takes a
- string of length one, i.e., a single character, and returns the corresponding
- ASCII value.
- >>> ord('A')
- 65
- >>> ord('B')
- 66
- >>> for ch in "ASCII = numbers":
- print(ord(ch))
- # scipy constants
- >>> import scipy.constants
- >>> dir(scipy.constants) # list of constants
- >>> from scipy.constants import g, G
- >>> g
- 9.80665
- >>> G
- 6.6743e-11
- # ruff - linter written in Rust
- # Install
- $ pip install ruff
- # Usage
- $ ruff check . # lint all files in the current directory
- $ ruff check path/to/code # lint all files in /path/to/code
- $ ruff check path/to/code/*.py # lint all .py files in /path/to/code
- $ ruff check path/to/code/to/file.py # lint file.py
- # virtual environment in google-colab (doesn't work as expected)
- !pip install virtualenv
- !virtualenv <name>
- !source /content/<name>/bin/activate
- # google colab environment setup
- 1. !pip list | grep mpi4py
- 2. !pip install mpi4py
- 3. !nvidia-smi
- # writing file in google-colab
- %%writefile program.py
- [Shift + Enter] to save the above file
- !mpiexec --allow-run-as-root -np 8 python program.py
- !mpiexec --allow-run-as-root --oversubscribe -np 8 python program.py
- # downloading files to your local file system from google colab
- from google.colab import files
- with open('example.txt', 'w') as f:
- f.write('some content')
- files.download('example.txt')
- # uploading files from your local file system to google colab
- from google.colab import files
- files.upload()
- # permanently install a module on google colab
- * to be able to interact with Google Drive's operating system
- import os, sys
- * drive is a module that allows us use Python to interact with google drive
- from google.colab import drive
- * mounting google drive allows us to work with its contents
- drive.mount('/content/gdrive')
- * the last three lines are what changes the path of the file
- nb_path = '/content/notebooks'
- os.symlink('/content/gdrive/My Drive/Colab Notebooks', nb_path)
- sys.path.insert(0, nb_path) # or append(nb_path)
- * install the module in the notebook directory permanently
- !pip install --target=$nb_path <module>
- * colab notebook
- from google.colab import drive
- drive.mount('/content/gdrive')
- import sys
- sys.path.append('/content/gdrive/My Drive/Colab Notebooks')
- import <module>
- # rules of functional programming
- At its core, functional programming is just programming with functions - pure
- mathematical functions. The result of a function depends only on the arguments,
- and there are no side effects, such as I/O or mutation of state. Programs are
- built by combining functions together.
- There are two main things you need to know to understand the concept:
- * Data is immutable: If you want to change data, such as an array, you
- return a new array with the changes, not the original.
- * Functions are stateless: Functions act as if for the first time, every
- single time! In other words, the function always gives the same return
- value for the same arguments.
- There are three best practices that you should generally follow:
- 1. Your functions should accept at least one argument.
- 2. Your functions should return data, or another function.
- 3. Don't use loops!
- Doing functional programming meaningfully in a language without higher-order
- fucntions (the ability to pass functions as arguments and return functions),
- lambdas (anonymous functions), and generics is difficult. Most modern languages
- have these, but there are differences in how well different languages support
- functional programming. The languages with the best support are called
- functional programming languages. These include Haskell, OCaml, F#, and Scala,
- which are statically typed, and the dynamically typed Erlang and Clojure.
- Recall that the result of a function depends only on its inputs. Alas, almost
- all programming languages have "features" that break this assumption. Null
- values, type case (instanceof), type casting, exceptions, side-effects, and the
- possibility of infinite recursion are trap doors that break equational reasoning
- and impair a programmer's ability to reason about the behavior or correctness of
- a program.
- # infix function example
- The function combines two functions into one, applying g to the output of f.
- def compose(g, f):
- return lambda x: g(f(x))
- # number of values
- 2**20 == 1 << 20
- # pytype
- While annotations are optional for pytype, it will check and apply them where
- present.
- type inference and checking:
- $ pytype program.py
- Generate type annotations in standalone files ("pyi files"), which can be
- merged back into the Python source with a provided merge-pyi tool.
- merging back inferred type information:
- $ merge-pyi -i program.py .pytype/pyi/program.pyi
- # pyqtgraph examples
- pyqtgraph includes an extensive set of examples that can be accessed by running
- >>> import pyqtgraph.examples
- >>> pyqtgraph.examples.run()
- # pip installation logs
- $ pip install pylint --log pylintlog.txt
- # install qtbase5-dev and set qmake tool on PATH for pyqt5
- $ sudo apt-get install qtbase5-dev
- $ which qmake
- /usr/bin/qmake
- $ qmake --version
- QMake version 3.1
- Using Qt version 5.15.2 in /usr/lib/i386-linux-gnu
- $ pip install pyqt5
- # pip install pyqt5
- Successfully installed PyQt-builder-1.13.0 packaging-21.3 ply-3.11
- pyparsing-3.0.9 setuptools-65.3.0 sip-6.6.2 toml-0.10.2
- Cleaning up ...
- Removing source in /tmp/pip-install-2gmr_frd/sip
- Removed build tracker: '/tmp/pip-req-tracker-vuj8lfsc'
- Installing build dependencies ... done
- Running command /usr/bin/python3 /tmp/tmppi_h1r7x
- get_requires_for_build_wheel /tmp/tmpxrg1n2t9
- Getting requirements to build wheel ... done
- Created temporary directory: /tmp/pip-modern-metadata-b___1na9b
- Running command /usr/bin/python3 /tmp/tmpwp46ffki
- prepare_metadata_for_build_wheel /tmp/tmpxrg1n2t9
- Querying qmake about your Qt installation ...
- This is the GPL version of PyQt 5.15.7 (licensed under the GNU General
- Public License) for Python 3.8.2 on linux.
- Type 'L' to view the license.
- Type 'yes' to accept the terms of the license.
- Type 'no' to decline the terms of the license.
- Solution:
- When pip does not have a wheel to work from, it attempts to compile from
- source. By passing a --config-settings argument to pip you can pass an
- argument to the configure.py which would be used during compilation.
- pyqt has an argument to automatically accept the license --confirm-license.
- However, pip expects the argument in a Key=value form so you need to pass
- --confirm-license= (i.e. no value) and it will work. It took a while (about
- 30+ min) but did finally get pyqt5 installed.
- $ pip install pyqt5 --config-settings --confirm-license= --verbose
- # using 'or' in if statement
- weather == "Good!" or weather == "Great!"
- weather in ("Good!", "Great!")
- (weather == "Good!") or ("Great!")
- # Object-Oriented Programming
- Functions bound to objects are known as methods.
- For example, where a string possesses methods designed to manipulate its
- sequence of characters, a NumPy array possesses methods for operating on
- the numerical data bound to that array.
- >>> string = "institute"
- >>> string.capitalize() # use the string-method 'capitalize'
- "Institute"
- >>> import numpy as np
- >>> array = np.array([[0, 1, 2], [3, 4, 5]])
- >>> array.sum() # use the array-method 'sum'
- 15
- An object can possess data, known as attributes, which summarize information
- about that object.
- For example, the array-attributes ndim and shape provide information about the
- indexing-layout of that array's numerical data.
- # accessing an object's attributes
- >>> array.ndim
- 2
- >>> array.shape
- (2, 3)
- # psutil
- Psutil provides complete access to system information.
- >>> import psutil
- >>> psutil.boot_time()
- >>> psutil.cpu_count()
- >>> psutil.cpu_freq()
- >>> psutil.cpu_stats()
- >>> psutil.cpu_times()
- >>> psutil.cpu_times_percent()
- >>> psutil.cpu_percent()
- >>> psutil.cpu_percent(interval=5, percpu=True)
- >>> psutil.version_info
- >>> psutil.swap_memory()
- >>> psutil.users()
- >>> psutil.net_connections()
- >>> psutil.net_if_addrs()
- >>> psutil.disk_partitions()
- # there's always better way of writing
- dxs = [-1, 1, 1, -1, -1, 1, 1, -1]
- don't write as:
- ddxs = [random.random() * 0.7 - 0.7/2 for i in range(len(dxs))]
- always write as:
- ddxs = [random.random() * 0.7 - 0.7/2 for _ in dxs]
- # smart if/else condition
- # always recommend
- def condition(x):
- return x if x > 0 else 0
- # not recommend
- def condition(x):
- return (x > 0) * x
- if x > 0, then (x > 0) == 1 and (x > 0) * x == x
- else (x > 0) == 0 and (x > 0) * x == 0
- # if-else in return statement
- def fibonacci(n):
- return fibonacci(n - 1) + fibonacci(n - 2) if n > 3 else 1
- # access python modules source code
- >>> import numpy as np
- >>> np.ones
- <function ones at 0x7faa039956c0>
- >>> np.ones.__code__
- <code object ones at 0x7faa065dbeb0, file "/home/saran/.envn/dsci/lib/python3.11/site-packages/numpy/core/numeric.py", line 136>
- # idiomatic and pythonic code [Reference: https://martinheinz.dev/blog/32]
- * In Python you have choice of using either 'is' or '==' for comparisons, where 'is'
- checks identity and '==' checks value.
- * Using 'is None', 'is True' or 'is False' isn't just about convention or improved
- readability though. It also improves performance, especially if you would use
- 'x is None' instead of 'x == None' inside loop.
- * # Bad
- try:
- page = urlopen(url)
- ...
- finally:
- page.close()
- # Good
- from contextlib import closing
- with closing(urlopen(url)) as page:
- ...
- * # Bad
- import os
- try:
- os.remove(path)
- except FileNotFoundError:
- pass
- # Good
- from contextlib import suppress
- with suppress(FileNotFoundError):
- os.remove(path)
- * Variable unpacking
- # first = 1, middle = [2, 3, 4], last = 5
- first, *middle, last = [1, 2, 3, 4, 5]
- # first = 1, middle = 2, rest = [3, 4, 5]
- first, second, *rest = [1, 2, 3, 4, 5]
- # name = "John", address = "Some Street", email = "john@mail.com"
- name, address, *_, email = ["John", "Some Street", "Credit Number", "john@mail.com"]
- # header_row -< first line
- # table_rows -< list of remaining lines
- header_row, *table_rows = open("filename").read().split("\n")
- # module information
- >>> import numpy
- >>> numpy
- <module 'numpy' from '/home/saran/.envn/dsci/lib/python3.11/site-packages/numpy/__init__.py'>
- >>> numpy.__name__
- 'numpy'
- >>> numpy.__doc__
- # python paths
- >>> import sysconfig
- >>> sysconfig.get_paths()
- {'stdlib': '/usr/local/lib/python3.11',
- 'platstdlib': '/home/saran/.envn/dsci/lib/python3.11',
- 'purelib': '/home/saran/.envn/dsci/lib/python3.11/site-packages',
- 'platlib': '/home/saran/.envn/dsci/lib/python3.11/site-packages',
- 'include': '/usr/local/include/python3.11',
- 'platinclude': '/usr/local/include/python3.11',
- 'scripts': '/home/saran/.envn/dsci/bin',
- 'data': '/home/saran/.envn/dsci'}
- # dask reference
- (venv)$ pip install dask
- (venv)$ pip install --upgrade "dask[distributed]" # enable distributed compute
- # using local cluster to start dask
- >>> import dask
- >>> from dask.distributed import Client
- >>> client = Client()
- <Client: 'tcp://127.0.0.1:34085' processes=4 threads=4, memory=7.67 GiB>
- # Python 3.12 support for the Linux perf profiler
- Reference: https://docs.python.org/3.12/howto/perf_profiling.html
- We can run perf to sample CPU stack traces at 9999 hertz:
- $ perf record -F 9999 -g -o perf.data python program.py
- Then we can use perf report to analyze the data:
- $ perf report --stdio -n -g
- # bit manipulation
- 1. multiplication
- a = 10
- a = a << 1 # multiply a by 2
- 2. division
- a = 10
- a = a >> 1 # divide a by 2
- 3. True/False
- x = 5
- if x & 1 == 1:
- print("x is an odd number")
- else:
- print("x is an even number")
- Using Bitwise AND operator:
- * The idea is to check whether the last bit of the number is set or not.
- * If last bit is set then the number is odd, otherwise even.
- * If a number is odd & (bitwise AND) of the number by 1 will be 1, because
- the last bit would already be set, otherwise it will give 0 as output.
- # print without for-looping
- >>> institute = ["Institute", "of", "Mathematical", "Sciences"]
- >>> print(*institute)
- Institute of Mathematical Sciences
- # Integer string conversion length limitation
- Reference: https://docs.python.org/3/library/stdtypes.html#integer-string-conversion-
- length-limitation
- def fibonacci(n: int) -> int:
- """compute nth fibonacci"""
- fib: list[int] = [0, 1, 1]
- for _ in range(1, n):
- fib[-1] = fib[0] + fib[1]
- fib[0], fib[1] = fib[1], fib[-1]
- return fib[0] if n == 0 else fib[1] if n == 1 else fib[-1]
- if __name__ == "__main__":
- N: int = 100000
- print(fibonacci(N))
- (venv)$ ./fibonacci.py
- Traceback (most recent call last):
- File "/home/saran/codelearn/python/./fibonaccid.py", line 23, in <module>
- print(fibonacci(N))
- ValueError: Exceeds the limit (4300 digits) for integer string conversion;
- use sys.set_int_max_str_digits() to increase the limit
- (venv)$ python -X int_max_str_digits=0 fibonacci.py
- # binary methods on integer types
- 1. Return the number of bits necessary to represent an integer in binary,
- excluding the sign and leading zeros:
- >>> n = -37
- >>> bin(n)
- '-0b100101'
- >>> n.bit_length()
- 6
- 2. Return the number of ones in the binary representation of the absolute
- value of the integer:
- >>> n = 19
- >>> bin(n)
- '0b10011'
- >>> n.bit_count()
- 3
- >>> (-n).bit_count()
- 3
- 3. Return an array of bytes representing an integer:
- >>> (1024).to_bytes(2, byteorder='big')
- b'\x04\x00'
- The default values can be used to conveniently turn an integer into a
- single byte object:
- >>> (65).to_bytes()
- b'A'
- 4. Return the integer represented by the given array of bytes:
- >>> int.from_bytes(b'\x00\x10', byteorder='big')
- 16
- >>> int.from_bytes(b'\x00\x10', byteorder='little')
- 4096
- >>> int.from_bytes(b'\xfc\x00', byteorder='big', signed=True)
- -1024
- >>> int.from_bytes(b'\xfc\x00', byteorder='big', signed=False)
- 64512
- >>> int.from_bytes([255, 0, 0], byteorder='big')
- 16711680
- Reference: https://docs.python.org/3/library/stdtypes.html#int.from_bytes
- # disassemble - disassembler for python bytecode
- >>> import math
- >>> import dis
- >>> dis.dis("math.pi")
- 0 0 RESUME 0
- 1 2 LOAD_NAME 0 (math)
- 4 LOAD_ATTR 2 (pi)
- 24 RETURN_VALUE
- >>> dis.dis("math.sin()")
- 0 0 RESUME 0
- 1 2 LOAD_NAME 0 (math)
- 4 LOAD_ATTR 3 (NULL|self + sin)
- 24 CALL 0
- 32 RETURN_VALUE
- >>> import numpy as np
- >>> dis.dis("np.zeros()")
- 0 0 RESUME 0
- 1 2 LOAD_NAME 0 (np)
- 4 LOAD_ATTR 3 (NULL|self + zeros)
- 24 CALL 0
- 32 RETURN_VALUE
- Reference: https://docs.python.org/3/library/dis.html
- # format specifiers
- Format specifiers may also contain evaluated expressions. This allows code such as:
- >>> width = 10
- >>> precision = 4
- >>> value = decimal.Decimal('12.34567')
- >>> f'result: {value:{width}.{precision}}'
- 'result: 12.35'
- Once expressions in a format specifier are evaluated (if necessary), format specifiers
- are not interpreted by the f-string evaluator. Just as in str.format(), they are
- merely passed in to the __format__() method of the object being formatted.
- Reference: https://peps.python.org/pep-0498/#format-specifiers
- # invoke library for managing shell-oriented subprocesses
- $ pip install invoke
- $ invoke --list # to see which tasks are available, you use the --list option
- $ invoke all
- $ invoke clean
- Reference:
- Getting started: https://docs.pyinvoke.org/en/stable/getting-started.html
- Invoking tasks:
- https://docs.pyinvoke.org/en/stable/concepts/invoking-tasks.html#how-tasks-run
- # ctypes
- create python array:
- nval = 10 # number of element
- arr = (ctypes.c_int * nval)() # array with nval integer elements
- for n in range(nval): # initialize
- arr[n] = n
- converting an existing list into array:
- arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
- nval = len(arr)
- arr = (ctypes.c_int * nval)(*arr)
- # numpy.ctypeslib.ndpointer
- An ndpointer instance is used to describe an ndarray in restype and argtypes
- specifications. This approach is more flexible than using, for example,
- POINTER(c_double), since several restrictions can be specified, which are
- verified upon calling the ctypes function. These include data type, number of
- dimensions, shape and flags. If a given array does not satisfy the specified
- restrictions, a TypeError is raised.
- numpy.ctypeslib.ndpointer(dtype=None, ndim=None, shape=None, flags=None)
- Parameters: dtype: data-type, optional
- Array data-type.
- ndim: int, optional
- Number of array dimensions.
- shape: tuple of ints, optional
- Array shape.
- flags: str or tuple of str
- Array flags; may be one or more of:
- * C_CONTIGUOUS/C/CONTIGUOUS
- * F_CONTIGUOUS/F/FORTRAN
- * OWNDATA/O
- * WRITEABLE/W
- * ALIGNED/A
- * WRITEBACKIFCOPY/X
- Returns: klass: ndpointer type object
- A type object, which is an _ndtpr instance containing dtype,
- ndim, shape and flags information.
- Raises: TypeError
- If a given array does not satisfy the specified restrictions.
- # np.double
- >>> np.double is np.float64
- True
- # append numpy arrays to a list
- Ex = []
- for n in range(nsteps):
- for k in range(100):
- ex[k] = ...
- Ex.append(ex.copy()) # list of numpy arrays
- # append numpy arrays to a numpy array
- Ex = np.empty((0, ex.shape[0]))
- for n in range(nsteps):
- for k in range(100):
- ex[k] = ...
- Ex = np.vstack((Ex, ex)) # numpy array of numpy arrays
- # reshape numpy array
- >>> ex = np.zeros(4*10, dtype=np.float64)
- >>> ex = ex.reshape(10, 4)
- >>> ex
- array([[0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.]])
- >>> ex = np.zeros(4*10, dtype=np.float64)
- >>> ex = ex.reshape(ex.shape[0], 1)
- >>> ex
- array([[0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.],
- [0.]])
- >>> ex = np.zeros(4*10, dtype=np.float64)
- >>> ex = np.reshape(ex, (-1, 4))
- >>> ex
- array([[0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.]])
- >>> ex = np.zeros(4*10, dtype=np.float64)
- >>> ex.shape = (ex.size//4, 4)
- >>> ex
- array([[0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.]])
- >>> ex = np.zeros(4*10, dtype=np.float64)
- >>> ex.shape = (-1, 4)
- >>> ex
- array([[0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.]])
- >>> ex = np.zeros(4*10, dtype=np.float64)
- >>> ex = np.reshape(ex, (1, ex.size))
- >>> ex
- array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
- 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
- 0., 0., 0., 0., 0., 0., 0., 0.]])
- >>> ex.shape
- (1, 40)
- >>> ex.size
- 40
- >>> ex = np.zeros(4*10, dtype=np.float64)
- >>> ex.shpae = (-1, 4)
- >>> ex
- array([[0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.]])
- >>> ex.flatten()
- array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
- 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
- 0., 0., 0., 0., 0., 0.])
- >>> ex
- array([[0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.]])
- # type hints
- Reference:
- https://docs.python.org/3/library/typing.html
- https://mypy.readthedocs.io/en/stable/cheat_sheet_py3.html
- Checking the type of a variable:
- x = [1, 2, 3, 4, 5] # list
- type(x) # returns list
- isinstance(x, list) # True
- Type aliases:
- A type alias is defined using the type statement, which creates an instance of
- TupeAliasType. In this example, Vector and list[float] will be treated equivalently
- by static type checkers:
- type Vector = list[float]
- def scale(scalar: float, vector: Vector) -> Vector:
- return [scalar * num for num in vector]
- passes type checking (a list of floats qualifies as a Vector):
- new_vector = scale(2.0, [1.0, -4.2, 5.4])
- note: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs [annotation-unchecked]
- $ mypy --check-untyped-defs *.py
- # print docstrings
- >>> import numpy as np
- >>> import sources
- >>> dir(np)
- >>> dir(sources)
- >>> help(np.zeros)
- >>> help(sources.pulse)
- >>> print(np.zeros.__doc__)
- >>> print(sources.pulse.__doc__)
- # measure execution time with timeit
- >>> import timeit
- >>> import numpy as np
- >>> import sources
- >>> ke = 200
- >>> ex = np.zeros(ke, dtype=np.float64)
- >>> hy = np.zeros(ke, dtype=np.float64)
- >>> timeit.timeit('sources.pulse(ke, ex, hy)', globals=globals())
- >>> timeit.timeit(lambda: sources.pulse(ke, ex, hy))
- >>> timeit.Timer(lambda: sources.pulse(ke, ex, hy)).timeit()
- Normally, such a function would run in a few milliseconds, but the reported
- timings are in the order of seconds. That's because, by default, timeit.timeit
- will run the benchmarked code 1 million times to provide a result where any
- temporary change in speed of the execution won't impact the final result much.
- The default value for number(number of executions) is 1,000,000. Be aware that
- running time-consuming code with the default value can take significant time.
- >>> timeit.timeit('sources.pulse(ke, ex, hy)', number=10, globals=globals())
- >>> timeit.timeit(lambda: sources.pulse(ke, ex, hy), number=10)
- >>> timeit.Timer(lambda: sources.pulse(ke, ex, hy)).timeit(number=10)
- timeit.repeat() to repeat the timeit() function, the result is returned as a list:
- >>> timeit.repeat(lambda: sources.pulse(ke, ex, hy), repeat=5, number=100)
- Reference: https://note.nkmk.me/en/python-timeit-measure/
- # profiling CPU usage
- $ python -m cProfile sources.py
- $ python
- >>> import cProfile
- >>> profile = cProfile.Profile()
- >>> import sources
- >>> import numpy as np
- >>> ke = 200
- >>> ex = np.zeros(ke, dtype=np.float64)
- >>> hy = np.zeros(ke, dtype=np.float64)
- >>> profile.runcall(lambda: sources.pulse(ke, ex, hy))
- >>> profile.print_stats()
- # profiling and timing script
- %time time the execution of a single statement
- %timeit time repeated execution of a single statement for more accuracy
- %prun run script with the profiler
- %lprun run script with the line-by-line profiler
- %memit measure the memory use of a single statement
- %mprun run code with the line-by-line memory profiler
- Note: the last four commands are not bundled with IPython - you'll need to get
- the line_profiler and memory_profiler extensions.
- Reference:
- https://jakevdp.github.io/PythonDataScienceHandbook/01.07-timing-and-profiling.html
- # timing script snippets with %time and %timeit
- In [1]: import numpy as np
- In [2]: ke = 200
- In [3]: ex = np.zeros(ke, dtype=np.float64)
- In [4]: hy = np.zeros(ke, dtype=np.float64)
- In [5]: import sources
- In [6]: %time sources.pulse(ke, ex, hy)
- In [7]: %timeit sources.pulse(ke, ex, hy)
- # profiling full script with %prun
- In [1]: import numpy as np
- In [2]: ke = 200
- In [3]: ex = np.zeros(ke, dtype=np.float64)
- In [4]: hy = np.zeros(ke, dtype=np.float64)
- In [5]: import sources
- In [6]: %prun sources.pulse(ke, ex, hy)
- # line-by-line profiling with %lprun
- In [1]: import numpy as np
- In [2]: ke = 200
- In [3]: ex = np.zeros(ke, dtype=np.float64)
- In [4]: hy = np.zeros(ke, dtype=np.float64)
- In [5]: import sources
- In [6]: %load_ext line_profiler # load the line_profiler IPython extension
- In [7]: %lprun -f sources.pulse sources.pulse(ke, ex, hy)
- # memory profiling with %memit and %mprun
- In [1]: import numpy as np
- In [2]: ke = 200
- In [3]: %load_ext memory_profiler
- In [4]: %memit ex = np.zeros(ke, dtype=np.float64)
- In [5]: %memit hy = np.zeros(ke, dtype=np.float64)
- In [6]: %mprun -f sources.pulse sources.pulse(ke, ex, hy)
- # pytest
- $ pytest
- $ pytest ./
- $ pytest file.py
- $ pytest file.py --collect-only
- $ pytest file.py::test_name
- $ pytest file.py -k <sub_string>/test_name
|