os.environ["HUGGINGFACEHUB_API_TOKEN"]="<Huggingface_API_Key>" # hugginface에서 가입 후 key 발급
# open-source LLM from Hugging Face
llm=HuggingFaceHub(repo_id="google/flan-t5-large")
llm_out=llm("Which is most expensive city in the world?")
print(llm_out)
/home/kubwa/anaconda3/envs/langchain/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
from .autonotebook import tqdm as notebook_tqdm
/home/kubwa/anaconda3/envs/langchain/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py:127: FutureWarning: '__init__' (from 'huggingface_hub.inference_api') is deprecated and will be removed from version '0.19.0'. `InferenceApi` client is deprecated in favor of the more feature-complete `InferenceClient`. Check out this guide to learn how to convert your script to use it: https://huggingface.co/docs/huggingface_hub/guides/inference#legacy-inferenceapi-client.
warnings.warn(warning_message, FutureWarning)
london
from langchain import PromptTemplate
# Write a query template
template = "Which is most {input} city in the world?"
# Create a prompt template
prompt = PromptTemplate(template=template, input_variables=['input'], )
#Format the prompt
_input=prompt.format(input="expensive")
# Generate the output
output = llm(_input)
# The response
print(output)
london
from langchain.output_parsers import CommaSeparatedListOutputParser
from langchain.prompts import PromptTemplate
# Initialize the parser
output_parser = CommaSeparatedListOutputParser()
# Create format instructions
format_instructions = output_parser.get_format_instructions()
# Create a prompt to request a list
prompt = PromptTemplate(
template="List three {subject}.\n{format_instructions}",
input_variables=["subject"],
partial_variables={"format_instructions": format_instructions}
)
# Define a query to prompt the model
query = "Planets in the Universe"
# Generate the output
output = llm(prompt.format(subject=query))
# Parse the output using the parser
parsed_result = output_parser.parse(output)
# The result is a list of items
print(parsed_result)
['uranus', 'neptune', 'and jupiter']
from langchain.document_loaders import PyPDFLoader
loader = PyPDFLoader("dataset/kb_23849_1_1.pdf")
transcript = loader.load_and_split()
from langchain.embeddings import HuggingFaceEmbeddings
embeddings = HuggingFaceEmbeddings(model_name = "sentence-transformers/all-MiniLM-L6-v2")
text_embeddings = embeddings.embed_query("Which is most expensive city in the world?")
print(text_embeddings)
from langchain.vectorstores import FAISS
db = FAISS.from_documents(docs, embeddings)
query = "Why Use Machine Learning?"
docs = db.similarity_search(query)
print(docs[0].page_content)
특별약관
제5장 배상책임 관련 특별약관
from langchain.chains import RetrievalQA
retriever = db.as_retriever()
qa = RetrievalQA.from_chain_type(
llm=llm,
chain_type="stuff",
retriever=retriever,
#return_source_documents=True,
#verbose=True
)
query = "What is Machine Learning?"
result = qa({"query": query})
print(result['result'])
not enough information
from langchain.retrievers.web_research import WebResearchRetriever
from langchain.utilities import GoogleSearchAPIWrapper
from langchain.chains import RetrievalQAWithSourcesChain
os.environ["GOOGLE_CSE_ID"] = "xxx"
os.environ["GOOGLE_API_KEY"] = "xxx"
search = GoogleSearchAPIWrapper()
web_research_retriever = WebResearchRetriever.from_llm(vectorstore=db, llm=llm, search=search)
qa_chain = RetrievalQAWithSourcesChain.from_chain_type(
llm=llm,
chain_type="stuff",
retriever=web_research_retriever,
#return_source_documents=True,
#verbose=True
)
result = qa_chain({"question": query})
print(result)
from langchain.document_loaders import UnstructuredPDFLoader
from langchain.indexes import VectorstoreIndexCreator
loader1 = [UnstructuredPDFLoader("MLInterview.pdf") for fn in os.listdir("/")]
llm=HuggingFaceHub(repo_id="google/flan-t5-large")
from langchain.text_splitter import CharacterTextSplitter
index = VectorstoreIndexCreator(
embedding=HuggingFaceEmbeddings(),
text_splitter=CharacterTextSplitter(chunk_size=1000, chunk_overlap=100)).from_loaders(loader1)
results = index.query("What is Machine Learning?", llm=llm)
print(results)
#results = index.query("\n When to use ensemble learning?", llm=llm)
#print(results)
Machine learning is a branch of computer science which deals with system programming in order to automatically learn
from langchain.agents import load_tools
from langchain.agents import initialize_agent
from langchain.agents import AgentType
tools = load_tools(["wikipedia"])
agent = initialize_agent(tools,
llm,
agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION,
verbose=True,
max_iterations=1,
handle_parsing_errors=True)
agent.run("Who is Barak Obama?")
[1m> Entering new AgentExecutor chain...[0m
[32;1m[1;3mAction: president of the united states[0m
Observation: Invalid Format: Missing 'Action Input:' after 'Action:'
Thought:[32;1m[1;3m[0m
[1m> Finished chain.[0m
'Agent stopped due to iteration limit or time limit.'
from langchain.tools import YouTubeSearchTool
tool = YouTubeSearchTool()
ans = tool.run(
"How to choose a Niche"
)
print(ans)
#%pip install duckduckgo-search
from langchain.tools import DuckDuckGoSearchRun
search = DuckDuckGoSearchRun()
search.run("Tell me about Abraham Lincoln")
"Abraham Lincoln ( / ˈlɪŋkən / LINK-ən; February 12, 1809 - April 15, 1865) was an American lawyer, politician, and statesman who served as the 16th president of the United States from 1861 until his assassination in 1865. 1809-1865 Who Was Abraham Lincoln? Abraham Lincoln was the 16 th president of the United States, serving from 1861 to 1865, and is regarded as one of America's greatest heroes due to his... ADVERTISEMENT Abraham Lincoln was 6 feet 4 inches tall, making him the tallest U.S. president in history. Who was Abraham Lincoln's wife? We Are Teachers; Wikimedia Commons In 1842, Abraham Lincoln married Mary Todd, the daughter of a prominent Kentucky slave-owning family. They lived in Springfield, Illinois, and had four sons. Lincoln Is One of the Four Assassinated US Presidents in History Abraham Lincoln facts reveal that not only was he one of the four US presidents (Lincoln, Garfield, McKinley, and Kennedy) who were assassinated, but that he was actually the first assassinated US president. Feb. 12, 1809, was the day Abraham Lincoln was born. Review his legacy and life during one of the most trying times in American history."
from langchain.prompts import ChatPromptTemplate
from langchain.schema import StrOutputParser
prompt = ChatPromptTemplate.from_messages([
("system", "You're a finance person who likes to roam around the world, and very well versed about the world economy."),
("human", "{question}")
])
runnable = prompt | llm
for chunk in runnable.stream({"question": "Which is most expensive city in the world?"}):
print(chunk, end="", flush=True)
New York
from langchain import LLMChain
chain = LLMChain(llm = llm, prompt = prompt)
print(chain.run(question="Which is most expensive city in the world?"))
New York
from langchain.chains import SimpleSequentialChain
template1 = "Which is most {input} city in the world?"
template2 = "Which is most {input} country in the world?"
prompt1 = PromptTemplate(template=template1, input_variables=['input'], )
prompt2 = PromptTemplate(template=template2, input_variables=['input'])
chain1 = LLMChain(llm = llm, prompt = prompt1)
chain2 = LLMChain(llm = llm, prompt = prompt2)
all_chains = SimpleSequentialChain(chains=[chain1, chain2], verbose=True)
print(all_chains.run("expensive"))
[1m> Entering new SimpleSequentialChain chain...[0m
[36;1m[1;3mlondon[0m
[33;1m[1;3mUnited Kingdom[0m
[1m> Finished chain.[0m
United Kingdom
from langchain.chains import ConversationChain
conversation = ConversationChain(llm=llm, verbose=True)
conversation.predict(input="I have a cat.")
conversation.predict(input="My mom gave me a cat and a dog.")
conversation.predict(input="Now how many pets I have?")
[1m> Entering new ConversationChain chain...[0m
Prompt after formatting:
[32;1m[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.
Current conversation:
Human: I have a cat.
AI:[0m
[1m> Finished chain.[0m
[1m> Entering new ConversationChain chain...[0m
Prompt after formatting:
[32;1m[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.
Current conversation:
Human: I have a cat.
AI: I have a cat.
Human: My mom gave me a cat and a dog.
AI:[0m
[1m> Finished chain.[0m
[1m> Entering new ConversationChain chain...[0m
Prompt after formatting:
[32;1m[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.
Current conversation:
Human: I have a cat.
AI: I have a cat.
Human: My mom gave me a cat and a dog.
AI: I have a dog.
Human: Now how many pets I have?
AI:[0m
[1m> Finished chain.[0m
'I have three pets.'
[1m> Entering new ConversationChain chain...[0m
Prompt after formatting:
[32;1m[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.
Current conversation:
Human: Hi There!
AI:[0m
[1m> Finished chain.[0m
'Hello!'
conversation.predict(input="I would like to know more about the world")
[1m> Entering new ConversationChain chain...[0m
Prompt after formatting:
[32;1m[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.
Current conversation:
Human: Hi There!
AI: Hello!
Human: I would like to know more about the world
AI:[0m
[1m> Finished chain.[0m
'Human: What would you like to know?'
conversation.predict(input="which is the most populated country in the world?")
[1m> Entering new ConversationChain chain...[0m
Prompt after formatting:
[32;1m[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.
Current conversation:
Human: Hi There!
AI: Hello!
Human: I would like to know more about the world
AI: Human: What would you like to know?
Human: which is the most populated country in the world?
AI:[0m
[1m> Finished chain.[0m
'Human: The most populated country in the world is China.'
conversation.memory
ConversationBufferMemory(chat_memory=ChatMessageHistory(messages=[HumanMessage(content='Hi There!'), AIMessage(content='Hello!'), HumanMessage(content='I would like to know more about the world'), AIMessage(content='Human: What would you like to know?'), HumanMessage(content='which is the most populated country in the world?'), AIMessage(content='Human: The most populated country in the world is China.')]))
from langchain.callbacks import FileCallbackHandler
from langchain.chains import LLMChain
from langchain.prompts import PromptTemplate
from loguru import logger
logfile = "output.log"
logger.add(logfile, colorize=True, enqueue=True)
handler = FileCallbackHandler(logfile)
prompt = PromptTemplate.from_template("Conversation:: + {message} = ")
chain = LLMChain(llm=llm, prompt=prompt, callbacks=[handler], verbose=True)
msg = chain.run(message="What is the capital of United States?")
logger.info("Answer:: " + msg)
[1m> Entering new LLMChain chain...[0m
Prompt after formatting:
[32;1m[1;3mConversation:: + What is the capital of United States? = [0m
[32m2024-02-20 17:57:32.528[0m | [1mINFO [0m | [36m__main__[0m:[36m<module>[0m:[36m15[0m - [1mAnswer:: United States Capitol, Washington, D.C.[0m
[1m> Finished chain.[0m