Regular Expressions

Basics
• \d   Matches any digit, e.g. 0-9, but also digits in foreign scripts     Example
  
  

  import re s = "I bought 10 pounds of potatoes for $5.27." re.findall(r'\d', s) ['1', '0', '5', '2', '7']

• \d+   Matches one or more digits     Example
  
  

  import re s = "I bought 10 pounds of potatoes for $5.27." re.findall(r'\d+', s) # + indicates "one or more" ['10', '5', '27']

• \d{1,3}   Matches 1-3 digits     Examples
  
  

  import re s = "January 20, 2021" re.findall(r'\d{1,3}', s) ['20', '202', '1'] import re s = "They spent $1,250,000 on 45,000 trees in 2021." re.findall(r'\d{1,3}(?:,\d\d\d)+', s) ['1,250,000', '45,000']

• \d{4,}   Matches at least 4 digits     Example
  
  

  import re s = "10000 people gathered on January 20, 2021." re.findall(r'\d{4,}', s) ['10000', '2021']

• \d{,4}   Matches up to 4 digits     Example
  
  

  import re s = "He spent $8000 in a budget of $11000." re.findall(r'\$\d{,4}', s) ['$8000', '$1100']

• \d{4}   Matches 4 digits     Example
  
  

  import re s = "10000 people gathered on January 20, 2021." re.findall(r'\b\d{4}\b', s) ['2021']

• \s   Matches a whitespace (e.g. regular space, tab, newline, no-break space, thin/wide spaces)     Example
  
  

  import re s = "A line with a\ttab." re.findall(r'\s', s) [' ', ' ', ' ', '\t']

• \S   Matches any non-whitespace     Example
  
  

  import re s = "2 cows." re.findall(r'\S', s) ['2', 'c', 'o', 'w', 's', '.']

• \S+   Matches one or more non-whitespace     Example
  
  

  import re s = "2 cows." re.findall(r'\S+', s) ['2', 'cows.']

• \s?   Matches zero or one whitespace (? indicates 0 or 1 of the preceding character or group; optional)     Example
  
  

  import re s = "Temperatures: 68F, 20 C" re.findall(r'\d+\s?[FC]\b', s) ['68F', '20 C']

• \s*   Matches zero or more whitespaces (* indicates 0 or more of the preceding character or group)     Example
  
  

  import re s = "John,Mary, and Paul" re.findall(r'\S,\s*\S', s) ['n,M', 'y, a']

• \s+   Matches one or more whitespaces (+ indicates 1 or more of the preceding character or group)     Example
  
  

  import re s = "John,Mary, and Paul" re.findall(r'\S,\s+\S', s) ['y, a']

• \w   Matches a word character (letter or digit)     Example
  
  

  import re s = '5 words: Dvořák, Москва المغرب 中国 भारत' re.findall(r'\w+', s) ['5', 'words', 'Dvořák', 'Москва', 'المغرب', '中国', 'भ', 'रत'] Note that word 'भारत' is split into 'भ', 'रत' . This is due to a vowel sign inside the Hindi word that does not count as a "word" character in regular expressions. See "Practical examples: Indian words" for how to fix this.

• [aeiou]   Matches any character inside the square brackets     Example
  
  

  import re s = "The clown laughed." re.findall(r'[aeiou]', s) ['e', 'o', 'a', 'u', 'e']

• [^aeiou]   Matches any character not inside the square brackets after initial ^     Example
  
  

  import re s = "The clown laughed." re.findall(r'[^aeiou]', s) ['T', 'h', ' ', 'c', 'l', 'w', 'n', ' ', 'l', 'g', 'h', 'd', '.']

• [1-9a-z]   Matches any character in a range.     Example
  
  

  import re s = "In 2021!" re.findall(r'[1-9a-z]', s) ['n', '2', '2', '1']

• .   Matches any character (except newline)     Examples
  
  

  import re s = "DSCI510 from 9-5 but not 24/7" re.findall(r'\d.\d', s) ['510', '9-5', '4/7'] import re s = "He can ran 100 meters in 10.35 seconds." re.findall(r'\d+\.\d+', s) # \. stands for an actual period ['10.35']

• +?   Repeats a character one or more times (non-greedy, i.e. as short as possible)     Examples
  
  

  import re s = "He came on March 15 and left on March 22, 2021." re.findall(r'(March.+\d+)', s) ['March 15 and left on March 22, 2021'] import re s = "He came on March 15 and left on March 22, 2021." re.findall(r'(March.+?\d+)', s) ['March 15', 'March 22']

• *?   Repeats a character zero or more times (non-greedy, i.e. as short as possible)     Examples
  
  

  import re s = '<a href="https://www.usc.edu" title="University of Southern California">USC<\a>' re.findall(r'".*"', s) ['"https://www.usc.edu" title="University of Southern California"'] Returns 1 string, probably not intended. import re s = '<a href="https://www.usc.edu" title="University of Southern California">USC<\a>' re.findall(r'".*?"', s) ['"https://www.usc.edu"', '"University of Southern California"'] Returns 2 strings, as intended.

• ^   Matches the beginning of a line     Example
  
  

  import re s = "10+20=30" re.findall(r'^\d+', s) ['10']

• $   Matches the end of a line     Examples
  
  

  import re s = "10+20=30" re.findall(r'\d+$', s) ['30'] import re s = "He spent $400 on a dinner for two on February 14." re.findall(r'\$\d+', s) # \$ stands for an actual dollar sign ['$400']

• \b   Matches word boundary     Examples
  
  

  The boundary marker prevents the F in Ford to match the F as an abbreviation for Fahrenheit. import re s = "99F in a 1999 Ford Mustang." re.findall(r'(\d+)\s*([CF])\b', s) [('99', 'F')] import re s = "99F in a 1999 Ford Mustang." re.findall(r'(\d+)\s*([CF])', s) [('99', 'F'), ('1999', 'F')]

• (...)   Captures a group     Example
  
  

  import re s = "68F is 20 C." re.findall(r'(\d+)\s*([CF])', s) [('68', 'F'), ('20', 'C')]

• (?:...)   Define a non-capture group     Example
  
  

  import re s = "They spent $1,250,000 on 45,000 trees in 2021." re.findall(r'\d{1,3}(?:,\d\d\d)+', s) ['1,250,000', '45,000']

• cat|dog   Alternatives     Examples
  
  

  Alternatives inside a capture group: import re s = "He visited us on June 27 and July 4." re.findall(r'(June|July|August)\s+([1-9]|[12]\d|3[01])\b', s) [('June', '27'), ('July', '4')] Alternatives inside a non-capture group: import re s = "He visited us on June 27 and July 4." re.findall(r'(?:June|July|August)\s+(?:[1-9]|[12]\d|3[01])\b', s) ['June 27', 'July 4'] Note that days of the month are limited to 1-31.

• r'...'   Interprets the string as a regular expression, i.e. without interpreting \ as a Python string escape character.     Examples
  
  

  With an r prefix, the \ inside a string is taken literally, not interpreted as the start of an escape sequence. s1 = '\b' s2 = r'\b' f'{s1} has {len(s1)} character; {s2} has {len(s2)} characters.' '\x08 has 1 character; \\b has 2 characters.' The r prefix is useful for regular expressions. (Note: \x08 represents the backspace control character.) In LaTeX, there is a \begin tag, which in Python has to be escaped as '\\begin' because in Python, '\b' is interpreted as a backspace. In a regular expression, \begin also has to be escaped, because '\b' is interpreted as a word-boundary mark. import re # string with single backslashes before begin: s = '\\begin{document} \\begin{section}' re.findall('(\\begin){([^{}]+)}', s) [] The above does not work (does not find anything), because Python interprets the \\begin part of a string as a 6-character string with a single \. The regular expression then interprets the resulting '\begin' as a word-boundary mark followed by the 4 letters egin . (Bad!) The following resolves the problem by doing a double escape. In Python, '\\\\begin' is the 7-character string \\begin (starting with two backslashes), as each '\\' is interpreted as the escape sequence for a \. The regular expression processor then interprets the \\ as an escape sequence for one literal \, which is what we intended. import re s = '\\begin{document} \\begin{section}' re.findall('(\\\\begin){([^{}]+)}', s) [('\\begin', 'document'), ('\\begin', 'section')] It works, but now the backslashes are getting out of hand, leading to "backslashitis": \\\\ Backslashitis can be controlled by using the r'' string prefix to suppress regular Python escape ineterpretation: import re s = r'\begin{document} \begin{section}' re.findall(r'(\\begin){([^{}]+)}', s) [('\\begin', 'document'), ('\\begin', 'section')]

• \1 \2   Backreferences that refer back to previously captured groups.     Examples
  
  

  Insert a space between a digit and a letter. import re s = "3cats and 5dogs" re.sub(r'([0-9])([a-z])', r'\1 \2', s) '3 cats and 5 dogs' \1 stands for the 1st captured group, \2 for the second captured group. Find adjacent duplicate digits. import regex s = "1233456666789" regex.findall(r'((\d)\2)', s) [('33', '3'), ('66', '6'), ('66', '6')] \1 stands for the 1st (= outer) captured group, \2 for second (= inner) captured group. Find adjacent duplicate words. import regex s = "He told me that that was impossible because of of the theory." regex.findall(r'((?:\pL\pM*)+)\s\1(?!\pL)', s) ['that', 'of'] \1 stands for the 1st captured group. \pL is any letter. \pM any letter modifier.

• Metacharacters .^*+?{}[]()\|$   To match a character that is also used as a metacharacter, escape it with a blackslash, etc. \. \$     Examples
  
  

  import re s = "He can ran 100 meters in 10.35 seconds." re.findall(r'\d+\.\d+', s) # \. stands for an actual period ['10.35'] import re s = "He spent $400 on a dinner for two on February 14." re.findall(r'\$\d+', s) # \$ stands for an actual dollar sign ['$400']

Lookarounds
• (?<=...)   Positive lookbehind: left context must match the ... inside (?<=...)     Example
  
  

  import re s = "On February 16, 2022, gas cost between $4.35 and $5 a gallon." re.findall(r'(?<=\$)(\d+(?:\.\d+)?)', s) ['4.35', '5']

• (?=...)   Positive lookahead: right context must match the ... inside (?=...)     Example
  
  

  import re s = "I paid 50 dollars for these 2 tickets." re.findall(r'(\d+(?:\.\d+)?)(?=\s*dollars?\b)', s) ['50']

• (?<!...)   Negative lookbehind: left context must NOT match the ... inside (?<!...)     Examples
  
  

  The following example mistakenly identifies the -15 in 12-15 as a negative number: import re s = "12-15" re.findall(r'-?\d+', s) ['12', '-15'] We can avoid a range "-" to be interpreted as a minus "-", by forbidding the number to be directly preceded by a digit: import re s = "12-15" re.findall(r'(?<!\d)-?\d+', s) ['12', '15'] Find words ending in -man but not in -human or -woman: import regex s = "chairman, demand, human, man, policeman, spokeswoman, subhuman, woman" regex.findall(r'\pL*(?<!hu|wo)man\b', s) ['chairman', 'man', 'policeman']

• (?!...)   Negative lookahead: right context must NOT match the ... inside (?!...)     Examples
  
  

  Looking for names that are not followed by Sr.: import re s = "Reagan, Bush Sr. and Clinton" re.findall(r'(\b[A-Z][a-z]{2,})\b(?! Sr\.)', s) ['Reagan', 'Clinton'] Find words starting with un- but not with under-: import regex s = "sunny, unable, unclear, under, underground, understand, undo, unfair" regex.findall(r'\bun(?!der)\pL*', s) ['unable', 'unclear', 'undo', 'unfair']

• More:   https://www.regular-expressions.info/lookaround.html   Lookarounds for regular expressions  
Methods
• search   Returns a Match-object if there is a match anywhere in the string.     Examples
  
  

  import re s = "He traveled on June 16 and on July 4." m = re.search(r'on (June|July) (\d+)', s) m.group(0, 1, 2) ('on June 16', 'June', '16') Returns a match object for the first match. m.group(0) covers the whole match, m.group(1) the 1st captured group, m.group(2) the 2nd captured group. import re s = "The temperature was 70°F." m = re.search(r'(-?\d+(?:\.\d+)?)\s*°\s*([CF])\b', s) m.group(0, 1, 2) ('70°F', '70', 'F')

• match   Returns a Match-object if there is a match at the beginning of the string.     Examples
  
  

  import re s = "From stephen.marquard@uct.ac.za Sat Jan 5 09:14:16 2008" m = re.match(r'From (\S+@\S+).*?(200[5-8]|2010)', s) m.group(0, 1, 2) ('From stephen.marquard@uct.ac.za Sat Jan 5 09:14:16 2008', 'stephen.marquard@uct.ac.za', '2008') Returns a match object. m.group(0) covers the whole match, m.group(1) the 1st captured group, m.group(2) the 2nd captured group. import re s = "From stephen.marquard@uct.ac.za Sat Jan 5 09:14:16 2008" re.match(r'(\S+@\S+).*?(200[5-8]|2010)', s) None No match, because the email address (\S+@\S+) is not at the beginning of the string.

• findall   Returns a list containing all matches.     Examples
  
  

  import re s = "I bought 10 pounds of potatoes for $5.27." re.findall(r'\d+', s) # + indicates "one or more" ['10', '5', '27'] import re s = "The temperature was 70°F (21.1°C)." re.findall(r'(-?\d+(?:\.\d+)?)\s*°\s*([CF])\b', s) [('70', 'F'), ('21.1', 'C')]

• sub   Substitute all parts of a string that match a regular expression with something else.     Examples
  
  

  The following substitution removes punctuation. import regex s = '• My fiancée (Mary) said: “hello”.' regex.sub(r'\pP', '', s) ' My fiancée Mary said hello' Insert a space between a digit and a letter. import re s = "3cats and 5dogs" re.sub(r'([0-9])([a-z])', r'\1 \2', s) '3 cats and 5 dogs' \1 stands for the 1st captured group, \2 for the second captured group.

• split   Returns a list where the string has been split at each match     Example
  
  

  Split string around commas and semicolons (incl. any following spaces) import re s = "apples, bananas,red onions; apple pie" re.split(r'[,;]\s*', s) ['apples', 'bananas', 'red onions', 'apple pie']

• compile   Compiles a regular expression into a reusable object. Might also speed up the code at run time.     Example
  
  

  import re re_float = re.compile(r'\d+\.\d+') (re_float.findall('$8.50 to $10.99 a pound'), re_float.findall('He is 1.88m.')) (['8.50', '10.99'], ['1.88'])

Flags
• re.IGNORECASE   "Apple" and "apple" considered the same.     Examples
  
  

  import re s = "The cat chased the mouse." re.findall(r'the', s, re.IGNORECASE) ['The', 'the'] Similarly, when using regex: import regex s = "The cat chased the mouse." regex.findall(r'the', s, regex.IGNORECASE) ['The', 'the']

Advanced package "regex"
• Script support   regex provides better script support than re     Examples
  
  

  import regex s = 'Dvořák Москва المغرب 中国 भारत' regex.findall(r'\p{Latin}+', s) ['Dvořák'] import regex s = 'Dvořák Москва المغرب 中国 भारत' regex.findall(r'\p{Cyrillic}+', s) ['Москва'] import regex s = 'Dvořák Москва المغرب 中国 भारत' regex.findall(r'\p{Arabic}+', s) ['المغرب'] import regex s = 'Dvořák Москва المغرب 中国 भारत' regex.findall(r'\p{InCJK_Unified_Ideographs}+', s) ['中国'] import regex s = 'Dvořák Москва المغرب 中国 भारत' regex.findall(r'\p{Devanagari}+', s) ['भारत']

• Punctuation support     Examples
  
  

  The following substitution removes punctuation. import regex s = '• My fiancée (Mary) said: “hello”.' regex.sub(r'\pP', '', s) ' My fiancée Mary said hello' Finds all quoted words import regex s = 'I said “hello” and he (François) said «bonjour».' regex.findall(r'\p{Pi}\pL+\p{Pf}', s) ['“hello”', '«bonjour»'] \p{Pi} stands for "opening quote", \p{Pf} for "closing quote"; \pL for "letter"

• Set operations   set intersection, union, difference; might require "Version1" flag set: (?V1)     Examples
  
  

  Set intersection: Latin lower-case letters import regex s = 'Dvořák Москва' regex.findall(r'(?V1)[\p{Latin}&&\p{Ll}]', s) ['v', 'o', 'ř', 'á', 'k'] Set difference: non-ASCII Latin import regex s = 'Dvořák Москва' regex.findall(r'(?V1)[\p{Latin}--\p{ASCII}]', s) ['ř', 'á']

• Character classes   \pL letter • \p{Ll} lower-case letter • \p{Lu} upper-case letter • \pM modifier character • \PL non-letter • \pP punctuation • \p{Pi} open quote • \p{Pf} close quote • \pS symbol • \pZ any space  
• More:   https://pypi.org/project/regex   PyPI regex module description  
• More:   https://www.regular-expressions.info/unicode.html   Unicode regular expressions  
• More:   https://regex101.com   Interactively explore regular expressions  
Practical examples
• Remove punctuation   including non-ASCII punctuation, but keeping non-ASCII letters     Example
  
  

  The following substitution removes punctuation. import regex s = '• My fiancée (Mary) said: “hello”.' regex.sub(r'\pP', '', s) ' My fiancée Mary said hello'

• Detect adjacent duplicate words.     Example
  
  

  Find adjacent duplicate words. import regex s = "He told me that that was impossible because of of the theory." regex.findall(r'((?:\pL\pM*)+)\s\1(?!\pL)', s) ['that', 'of'] \1 stands for the 1st captured group. \pL is any letter. \pM any letter modifier.

• Numbers   integers and floating point numbers (ASCII)     Example
  
  

  import re s = "100 -10C +10.5C 0 0.1 .2 3.14159 January 12-14 192.0.2.1 000 0." re.findall(r'(?<![0-9.])[-+]?(?:[1-9][0-9]*\.[0-9]+|0?\.[0-9]+|[1-9][0-9]*|0)(?!\.?[0-9])', s) ['100', '-10', '+10.5', '0', '0.1', '.2', '3.14159', '12', '14', '0'] Includes some context sensitivity to exclude things such as IP addresses (192.0.2.1) and negative numbers from ranges (12-14).

• Numbers with commas   125,500     Example
  
  

  import re s = "They spent $1,250,000 on 45,000 trees in 2021." re.findall(r'\d{1,3}(?:,\d\d\d)+', s) ['1,250,000', '45,000']

• Hybrid numbers (ASCII)   3.4 billion     Example
  
  

  Find and map hybrid numbers ("2 million") to numeric (2000000) import re def combine_hybrid_number_components(tuple): # tuple: ('2', 'million') d = {'million':1000000, 'billion':1000000000, 'trillion':1000000000000} product = float(tuple[0]) * d.get(tuple[1].lower()) return int(product) if product.is_integer() else product s = "Sales in 2021 were $3.4 BILLION, up $200 million from the year before." h = re.findall(r'(\d+(?:\.\d+)?)\s*((?:m|b|tr)illion)\b', s, re.IGNORECASE) list(map(combine_hybrid_number_components, h)) [3400000000, 200000000]

• Indian words (with letter modifiers)   भारत     Examples
  
  

  Many Indian languages use letter modifiers (\pM) such as vowel signs and viramas that need to be added when searching for words: import regex s = "नई दिल्ली से जर्मनी तक." regex.findall(r'(?:\pL\pM*)+', s) ['नई', 'दिल्ली', 'से', 'जर्मनी', 'तक'] Without letter modifiers, things fall apart. import regex s = "नई दिल्ली से जर्मनी तक." regex.findall(r'\pL+', s) ['नई', 'द', 'ल', 'ल', 'स', 'जर', 'मन', 'तक']

• Simple glosser   Spanish to English     Example
  
  

  This simple glossing program translates a sentence word-by-word, without context, using a simple dictionary. import regex spa_eng_dict = {"amigos": "friends", "hermana": "sister", "mi": "my", "pequeña": "little", "tiene": "has"} s = "María (mi hermana pequeña) tiene 1000 amigos." rest = s result = '' while m3 := regex.match(r'(.*?)(\pL+)(.*)', rest): # Capture pre-word (minimal), word, post-word. pre_word, word, rest = m3.group(1, 2, 3) # post-word becomes new rest for next iteration. result += pre_word + spa_eng_dict.get(word, word) result += rest # Add remaining text (e.g. after the last word). result 'María (my sister little) has 1000 friends.' The program uses a widely applicable idiom of regular expression, where a pattern of interest (here "\pL+" (= letters)) is preceded by a non-greedy pre-pattern string (.*?) and followed by a greedy post-pattern catch-the-rest (.*). The function preserves non-words such as spaces, punctuation, digits as well as words not covered by the dictionary (e.g. "María"). In the real world, the content of spa_eng_dict would be read in from a much larger dictionary file.

Methods related to strings
• str.isalpha()   Checks if string is all letters.     Examples
  
  

  'abc'.isalpha() True '12'.isalpha() False Chinese letters ("China") '中国'.isalpha() True 'T-cell'.isalpha() False False because of the hyphen ('-') which is not a letter.

• str.isdigit()   Checks if string is all digits.     Examples
  
  

  '12'.isdigit() True Arabic digits ("50") '٥٠'.isdigit() True '2.5'.isalpha() False False because of the period ('.') which is not a digit. Chinese number ("ten two" = 12) '十二'.isdigit() False False because Chinese numbers are not digits.

• str.islower()   Checks if string is all lower case.     Examples
  
  

  'a'.islower() True Greek letter beta 'β'.islower() True

• str.isupper()   Checks if string is all upper case.     Examples
  
  

  'A'.isupper() True Cyrillic capital letter Sha 'Ш'.isupper() True

• str.isspace()   Checks if string is all spaces.     Examples
  
  

  ' '.isspace() True tab is a space character '\t'.isspace() True

• str.startswith()   Checks if string starts with a given substring.     Example
  
  

  'that'.startswith('th') True

• str.endswith()   Checks if string ends with a given substring.     Example
  
  

  'that'.endswith('th') False

• str.strip()   Strips the beginning or end of a string of given characters (default: spaces).     Examples
  
  

  ' Hello ! '.strip() 'Hello !' '** Hello ! **'.strip(' *') 'Hello !'

• str.lstrip()   Strips the beginning of a string of given characters (default: spaces).     Example
  
  

  ' Hello ! '.lstrip() 'Hello ! '

• str.rstrip()   Strips the end of a string of given characters (default: spaces).     Example
  
  

  ' Hello ! '.rstrip() ' Hello !'

• str.find()   Returns the position of a substring inside a string.     Examples
  
  

  'Waldo'.find('ld') 2 'Waldo'.find('x') -1

• str.lower()   Returns the string in all lower case.     Example
  
  

  'NASA Lab'.lower() 'nasa lab'

• str.upper()   Returns the string in all upper case.     Example
  
  

  'The NASA lab'.upper() 'THE NASA LAB'

• str.capitalize()   Returns the string in capitalized case (first letter upper case).     Example
  
  

  'the NASA lab'.capitalize() 'The nasa lab' Upper case for first letter, lower case for rest.

• str.replace()   Replaces all instances of a given substring with another substring.     Example
  
  

  'art is art'.replace('art', 'love') 'love is love'

Functions related to lists
• enumerate()   adds a counter to an iterable (e.g. list)     Examples
  
  

  fruits = ["apple", "banana", "cherry"] list(enumerate(fruits)) [(0, 'apple'), (1, 'banana'), (2, 'cherry')] Useful in "for" loops: for i, fruit in enumerate(fruits): ... Optional second argument: enumeration start index months = ("January", "February", "March", "April") list(enumerate(months, 1)) # start enumeration at 1 [(1, 'January'), (2, 'February'), (3, 'March'), (4, 'April')] Useful in "for" loops: for i, month in enumerate(months, 1): ...

• map()   executes a specified function for each item in an iterable (e.g. list)     Examples
  
  

  Map all strings in list to lower case. l = ["My", "NASA", "project"] list(map(str.lower, l)) # str is the type for strings. ['my', 'nasa', 'project'] Map all units in list from singular to plural form. def eng_plural(s: str) -> str: irreg_eng_plurals = {"foot": "feet", "mouse": "mice"} if irregular_plural := irreg_eng_plurals.get(s): return irregular_plural elif s.endswith('y'): return s[:-1] + 'ies' elif s[-1:] in ['s', 'x'] or s[-2:] in ['ch', 'sh']: return s + 'es' else: return s + 's' units = ["inch", "foot", "yard", "mile"] list(map(eng_plural, units)) # str is the type for strings. ['inches', 'feet', 'yards', 'miles'] Find and map hybrid numbers ("2 million") to numeric (2000000) import re def combine_hybrid_number_components(tuple): # tuple: ('2', 'million') d = {'million':1000000, 'billion':1000000000, 'trillion':1000000000000} product = float(tuple[0]) * d.get(tuple[1].lower()) return int(product) if product.is_integer() else product s = "Sales in 2021 were $3.4 BILLION, up $200 million from the year before." h = re.findall(r'(\d+(?:\.\d+)?)\s*((?:m|b|tr)illion)\b', s, re.IGNORECASE) list(map(combine_hybrid_number_components, h)) [3400000000, 200000000]

• zip()   aggregates elements from each of the iterable (e.g. list) arguments into a tuple     Example
  
  

  husbands = ("Alexander", "Ben", "Charles") wives = ("Abigail", "Barbara", "Christine") list(zip(husbands, wives)) [('Alexander', 'Abigail'), ('Ben', 'Barbara'), ('Charles', 'Christine')]

urllib
• Description   urllib is a package that collects several modules for working with URLs: (1) urllib.request for opening and reading URLs (2) urllib.error containing the exceptions raised by urllib.request and (3) urllib.parse for parsing URLs  
• Examples   Romeo poem (print and count words)     Examples
  
  

  import urllib.request, urllib.parse, urllib.error lines = [] with urllib.request.urlopen('http://data.pr4e.org/romeo.txt') as f: for line in f: lines.append(line.decode().strip()) lines ['But soft what light through yonder window breaks', 'It is the east and Juliet is the sun', 'Arise fair sun and kill the envious moon', 'Who is already sick and pale with grief'] import urllib.request, urllib.parse, urllib.error word_counts = {} with urllib.request.urlopen('http://data.pr4e.org/romeo.txt') as f: for line in f: words = line.decode().split() for word in words: word_counts[word] = word_counts.get(word, 0) + 1 sorted(word_counts.items(), key=lambda x: x[1], reverse=True) [('is', 3), ('the', 3), ('and', 3), ('sun', 2), ('But', 1), ('soft', 1), ('what', 1), ('light', 1), ('through', 1), ('yonder', 1), ('window', 1), ('breaks', 1), ('It', 1), ('east', 1), ('Juliet', 1), ('Arise', 1), ('fair', 1), ('kill', 1), ('envious', 1), ('moon', 1), ('Who', 1), ('already', 1), ('sick', 1), ('pale', 1), ('with', 1), ('grief', 1)]

requests
• Description   A simple, yet elegant HTTP library for Python. Text extracted by requests is already decoded, i.e. converted from UTF-8 (Python bytes) to Unicode (Python string).  
• Examples   Romeo poem, JSON file, Chinese     Examples
  
  

  import requests r = requests.get('http://data.pr4e.org/romeo.txt') r.text 'But soft what light through yonder window breaks\nIt is the east and Juliet is the sun\nArise fair sun and kill the envious moon\nWho is already sick and pale with grief\n' import requests r = requests.get('http://data.pr4e.org/romeo.txt') r.text.strip().split('\n') ['But soft what light through yonder window breaks', 'It is the east and Juliet is the sun', 'Arise fair sun and kill the envious moon', 'Who is already sick and pale with grief'] Retrieve a document in JSON format. import requests r = requests.get('https://reqres.in/api/products/3') r.json() {'data': {'id': 3, 'name': 'true red', 'year': 2002, 'color': '#BF1932', 'pantone_value': '19-1664'}, 'support': {'url': 'https://reqres.in/#support-heading', 'text': 'To keep ReqRes free, contributions towards server costs are appreciated!'}} Returns a JSON object (a special type of Python dictionary). import regex import requests r = requests.get('https://en.wiktionary.org/wiki/%E5%86%AC%E5%A4%A9') set(regex.findall(r'\p{InCJK_Unified_Ideographs}*冬天\p{InCJK_Unified_Ideographs}*', r.text)) {'冬天價', '冬天裡', '核冬天', '冬天時', '冬天', '冬天道', '冬天頭', '冬天家'} requests already delivers properly decoded text.

beautifulsoup4
• Description   Beautiful Soup is a library that makes it easy to scrape information from web pages. It sits atop an HTML or XML parser, providing Pythonic idioms for iterating, searching, and modifying the parse tree.  
• Installation   pip install beautifulsoup4  
• Documentation   https://www.crummy.com/software/BeautifulSoup/bs4/doc  
• Example   Navigating inside the soup     Example
  
  

  from bs4 import BeautifulSoup html_doc = """ <html><head><title>The Dormouse's story</title></head> <body> <p class="title"><b>The Dormouse's story</b></p> <p class="story">Once upon a time there were three little sisters; and their names were <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>, <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>; and they lived at the bottom of a well.</p> <p class="story">...</p> """ soup = BeautifulSoup(html_doc, 'html.parser') { "soup.title": soup.title, "soup.title.string": soup.title.string, "soup.p": soup.p, "soup.p.get('class')": soup.p.get('class'), "soup.a": soup.a, "soup.a.attrs": soup.a.attrs, "soup.find_all('a')": soup.find_all('a'), "soup.a.parent.get('class')": soup.a.parent.get('class'), "number of soup.body.children": len(list(soup.body.children)) } {'soup.title': <title>The Dormouse's story</title>, 'soup.title.string': "The Dormouse's story", 'soup.p': <p class="title"><b>The Dormouse's story</b></p>, "soup.p.get('class')": ['title'], 'soup.a': <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>, 'soup.a.attrs': {'href': 'http://example.com/elsie', 'class': ['sister'], 'id': 'link1'}, "soup.find_all('a')": [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>, <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>, <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>], "soup.a.parent.get('class')": ['story'], 'number of soup.body.children': 7}

• Example   Find all 'academic' links on USC webpage     Example
  
  

  import urllib.request, urllib.parse, urllib.error from bs4 import BeautifulSoup url = "https://www.usc.edu" html = urllib.request.urlopen(url).read() soup = BeautifulSoup(html, 'html.parser') # Retrieve all of the anchor tags tags = soup('a') links = [] for tag in tags: if (link := tag.get('href')) and 'academic' in link: links.append(link) links ['http://about.usc.edu/administration/academic-deans/', 'https://academics.usc.edu/', 'http://academics.usc.edu/calendar/']

SQL
• CREATE   TABLE TableName (attribute1 Type1, attribute2 Type2)     Example
  
  

  import sqlite3 conn = sqlite3.connect('emaildb.sqlite') # 'emaildb.sqlite' is filename where database will be stored. cur = conn.cursor() cur.execute('DROP TABLE IF EXISTS Counts') cur.execute('CREATE TABLE Counts (email TEXT, count INTEGER)') cur.close() None

• INSERT   INTO TableName (attribute1, attribute2) VALUES (value1, value2)     Example
  
  

  import sqlite3 conn = sqlite3.connect('emaildb.sqlite') cur = conn.cursor() cur.execute('DROP TABLE IF EXISTS Counts') cur.execute('CREATE TABLE Counts (email TEXT, count INTEGER)') cur.execute("INSERT INTO Counts (email, count) VALUES ('stephen.marquard@uct.ac.za', 1)") cur.execute("INSERT INTO Counts (email, count) VALUES ('gsilver@umich.edu', 1)") list(cur.execute("SELECT email, count FROM Counts")) # cur.close() # uncomment in real app [('stephen.marquard@uct.ac.za', 1), ('gsilver@umich.edu', 1)]

• UPDATE   TableName SET attribute1 = value1 WHERE attribute2 = 'value2'     Example
  
  

  import sqlite3 conn = sqlite3.connect('emaildb.sqlite') cur = conn.cursor() cur.execute('DROP TABLE IF EXISTS Counts') cur.execute('CREATE TABLE Counts (email TEXT, count INTEGER)') cur.execute("INSERT INTO Counts (email, count) VALUES ('stephen.marquard@uct.ac.za', 1)") cur.execute("INSERT INTO Counts (email, count) VALUES ('gsilver@umich.edu', 1)") cur.execute("UPDATE Counts SET count = count + 1 WHERE email = 'stephen.marquard@uct.ac.za'") list(cur.execute("SELECT email, count FROM Counts")) # cur.close() # uncomment in real app [('stephen.marquard@uct.ac.za', 2), ('gsilver@umich.edu', 1)]

• SELECT   attribute1, attribute2 FROM TableName WHERE attribute1 = 'value1'     Example
  
  

  import sqlite3 conn = sqlite3.connect('emaildb.sqlite') cur = conn.cursor() cur.execute('DROP TABLE IF EXISTS Counts') cur.execute('CREATE TABLE Counts (email TEXT, count INTEGER)') cur.execute("INSERT INTO Counts (email, count) VALUES ('stephen.marquard@uct.ac.za', 1)") cur.execute("INSERT INTO Counts (email, count) VALUES ('gsilver@umich.edu', 1)") list(cur.execute("SELECT email, count FROM Counts ORDER BY count DESC LIMIT 10")) # cur.close() # uncomment in real app [('stephen.marquard@uct.ac.za', 1), ('gsilver@umich.edu', 1)]

• DELETE   FROM TableName WHERE attribute1 = 'value1'     Example
  
  

  import sqlite3 conn = sqlite3.connect('emaildb.sqlite') cur = conn.cursor() cur.execute('DROP TABLE IF EXISTS Counts') cur.execute('CREATE TABLE Counts (email TEXT, count INTEGER)') cur.execute("INSERT INTO Counts (email, count) VALUES ('stephen.marquard@uct.ac.za', 1)") cur.execute("INSERT INTO Counts (email, count) VALUES ('gsilver@umich.edu', 1)") cur.execute("DELETE FROM Counts WHERE email = 'gsilver@umich.edu'") list(cur.execute("SELECT email, count FROM Counts")) # cur.close() # uncomment in real app [('stephen.marquard@uct.ac.za', 1)]

• DROP   TABLE IF EXISTS TableName     Example
  
  

  import sqlite3 conn = sqlite3.connect('emaildb.sqlite') cur = conn.cursor() cur.execute('DROP TABLE IF EXISTS Counts') cur.close() None

Note: This is an early version of work in progress. Suggestions are very welcome.