Name – RUSHABH SHAH

Sem –5th
Roll No –19BCP108
Compiler Design

Lab2

Output:

Input:
a=int(input())
b=a+5
#this is the comment
print(a)

Output:
LexToken(NAME,'a',1,0)
LexToken(EQUAL,'=',1,1)
LexToken(NAME,'int',1,2)
LexToken(LPAREN,'(',1,5)
LexToken(NAME,'input',1,6)
LexToken(LPAREN,'(',1,11)
LexToken(RPAREN,')',1,12)
LexToken(RPAREN,')',1,13)
LexToken(NAME,'b',2,15)
LexToken(EQUAL,'=',2,16)
LexToken(NAME,'a',2,17)
LexToken(PLUS,'+',2,18)
LexToken(INT,5,2,19)
LexToken(NAME,'print',4,22)
LexToken(LPAREN,'(',4,27)
LexToken(NAME,'a',4,28)
LexToken(RPAREN,')',4,29)
number of tokens: 17

Code:

Indentifier.py
import ply.lex as lex
from countingToken import countLexToken

tokens = (
'INT',
'FLOAT',
'PLUS',
'MINUS',
'NUMBERSTRING',
'MULTIPLICATION',
'DIVISON',
'MODULO',
'LPAREN',
'NEWLINE',
 'NAME',
'INAME',
'COMMA',
'RPAREN',
'DOUBLEQUOTES',
'EQUAL',
'COMMENT',
)

t_ignore = '\t '

t_PLUS = r'\+'
t_MINUS = r'-'
t_MULTIPLICATION = r'\*'
t_DIVISON = r'\/'
t_MODULO = r'\%'
t_LPAREN = r'\('
t_RPAREN = r'\)'
t_COMMA = r','
t_EQUAL = r'\='

def t_INT(t):
r'\d+'
return t

def t_INAME(t):
r'[0-9]+[a-zA-Z]+[0-9]*'
print("Compile time error.")
print("Inllegle character '%s'", t.value[0])
t.lexer.skip(1)

def t_NAME(t):
r'[a-zA-Z][a-zA-Z_0-9]*'
t.type="NAME"
return t

def t_FLOAT(t):
r'\d+\.\d+'
return t

def t_NEWLINE(t):
r'\n+'
t.type = "NEWLINE"
return t

def t_DOUBLEQUOTES(t):
r'".*"'
t.value = t.value
 return t

def t_COMMENT(t):
r"\#.*"

def t_error(t):
print("Illegal character", t)
t.lexer.skip(1)

# \n is mandatory
lexer = lex.lex()
data = 'a = int(input()) \n b = a + 5 \n #this is the comment \n print(a)'

lexer.input(data)

# REMOVING COMMENT FROM HERE

strAfterRemovingComment = ""
while True:
tok = lexer.token()
if not tok:
break
strAfterRemovingComment += tok.value

#WRITING STRING AFTER REMOVING COMMENT

f = open("input.txt", "w")
f.write(strAfterRemovingComment)
f.close()

#COUNT TOKEN FROM ASSIGNMENT 1 METHOD

countLexToken()

coutingToken.py
import ply.lex as lex

def countLexToken():
tokens = (
'INT',
'FLOAT',
'PLUS',
'MINUS',
'NUMBERSTRING',
'MULTIPLICATION',
'DIVISON',
'MODULO',
'LPAREN',
 'NAME',
'INAME',
'COMMA',
'RPAREN',
'DOUBLEQUOTES',
'EQUAL',
)

t_PLUS = r'\+'
t_MINUS = r'-'
t_MULTIPLICATION = r'\*'
t_DIVISON = r'\/'
t_MODULO = r'\%'
t_LPAREN = r'\('
t_RPAREN = r'\)'
t_COMMA = r','
t_EQUAL = r'\='

def t_INT(t):
r'\d+'
t.value = int(t.value)
return t

def t_INAME(t):
r'[0-9]+[a-zA-Z]+[0-9]*'
print("Compile time error.")
print("Inllegle character '%s'", t.value[0])
t.lexer.skip(1)

def t_NAME(t):
r'[a-zA-Z][a-zA-Z_0-9]*'
t.type="NAME"
return t

def t_FLOAT(t):
r'\d+\.\d+'
t.value = float(t.value)
return t

def t_newline(t):
r'\n+'
t.lexer.lineno += len(t.value)

def t_DOUBLEQUOTES(t):
r'".*"'
t.value = t.value
return t
t_ignore = ' \t'

def t_error(t):
print("Illegal character", t)
t.lexer.skip(1)

lexer = lex.lex()
f = open("input.txt", "r")
strWithoutComment = f.read()
data = strWithoutComment
lexer.input(data)
count = 0
while True:
tok = lexer.token()
if not tok:
break
count += 1
print(tok)

print("number of tokens: ",count)