import json

# Load all the json files into a list for easy access and searching
file_paths = [
    "/mnt/data/extracted_posts_zk.json",
    "/mnt/data/extracted_posts_sharding.json",
    "/mnt/data/extracted_posts_l2.json",
    "/mnt/data/extracted_posts_pos.json",
    "/mnt/data/extracted_posts_economics.json"
]

posts_data = []

for file_path in file_paths:
    with open(file_path, 'r') as file:
        posts_data.extend(json.load(file))

# Display the total number of posts loaded
len(posts_data)

# Searching through the posts data for information about upcoming features in Ethereum

# Keywords for searching
keywords = ["upcoming features", "future developments", "next steps", "planned upgrades", "implementation roadmap"]

# Function to search for relevant posts
def search_posts(posts, keywords):
    relevant_posts = []
    for post in posts:
        for keyword in keywords:
            if keyword in post["content"].lower():
                relevant_posts.append(post)
    return relevant_posts

# Perform the search
upcoming_features_posts = search_posts(posts_data, keywords)

# Extract titles and a brief summary from the relevant posts
upcoming_features_info = [{"title": post["title"], "summary": post["content"][:150] + "..."} for post in upcoming_features_posts]

# Limit the output to a manageable number
upcoming_features_info[:5]  # Displaying the first 5 for brevity


---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[2], line 16
     13     return relevant_posts
     15 # Perform the search
---> 16 upcoming_features_posts = search_posts(posts_data, keywords)
     18 # Extract titles and a brief summary from the relevant posts
     19 upcoming_features_info = [{"title": post["title"], "summary": post["content"][:150] + "..."} for post in upcoming_features_posts]

Cell In[2], line 11, in search_posts(posts, keywords)
      9 for post in posts:
     10     for keyword in keywords:
---> 11         if keyword in post["content"].lower():
     12             relevant_posts.append(post)
     13 return relevant_posts

TypeError: string indices must be integers


# Adjusting the approach to correctly handle the structure of the posts data

def search_for_features(posts, keywords):
    relevant_info = []
    for post in posts:
        if any(keyword in post.get('content', '').lower() for keyword in keywords):
            relevant_info.append({
                "title": post.get("title", "No title"),
                "summary": post.get("content", "")[:150] + "..."
            })
    return relevant_info

# Search for upcoming features in Ethereum
upcoming_features_info = search_for_features(posts_data, keywords)

# Display the first 5 results for a brief overview
upcoming_features_info[:5]

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[3], line 14
     11     return relevant_info
     13 # Search for upcoming features in Ethereum
---> 14 upcoming_features_info = search_for_features(posts_data, keywords)
     16 # Display the first 5 results for a brief overview
     17 upcoming_features_info[:5]

Cell In[3], line 6, in search_for_features(posts, keywords)
      4 relevant_info = []
      5 for post in posts:
----> 6     if any(keyword in post.get('content', '').lower() for keyword in keywords):
      7         relevant_info.append({
      8             "title": post.get("title", "No title"),
      9             "summary": post.get("content", "")[:150] + "..."
     10         })
     11 return relevant_info

Cell In[3], line 6, in <genexpr>(.0)
      4 relevant_info = []
      5 for post in posts:
----> 6     if any(keyword in post.get('content', '').lower() for keyword in keywords):
      7         relevant_info.append({
      8             "title": post.get("title", "No title"),
      9             "summary": post.get("content", "")[:150] + "..."
     10         })
     11 return relevant_info

AttributeError: 'str' object has no attribute 'get'


# Re-evaluating the structure of the data in the posts to correctly extract information

# Checking the structure of a single post to understand how to access its contents
sample_post = posts_data[0]
sample_post_structure = {key: type(value) for key, value in sample_post.items()}

sample_post_structure

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[4], line 5
      1 # Re-evaluating the structure of the data in the posts to correctly extract information
      2 
      3 # Checking the structure of a single post to understand how to access its contents
      4 sample_post = posts_data[0]
----> 5 sample_post_structure = {key: type(value) for key, value in sample_post.items()}
      7 sample_post_structure

AttributeError: 'str' object has no attribute 'items'


# Parsing each string in the posts data as JSON to access their contents

def parse_json_strings(posts):
    parsed_posts = []
    for post_str in posts:
        try:
            post = json.loads(post_str)
            parsed_posts.append(post)
        except json.JSONDecodeError:
            # Skip any strings that can't be parsed as JSON
            continue
    return parsed_posts

# Parse the posts data
parsed_posts_data = parse_json_strings(posts_data)

# Check the structure of a single parsed post
if parsed_posts_data:
    sample_parsed_post_structure = {key: type(value) for key, value in parsed_posts_data[0].items()}
else:
    sample_parsed_post_structure = "No valid posts found"

sample_parsed_post_structure

# Examining the raw content of one of the posts to understand its format

# Displaying the first few characters of the first post to understand its format
sample_raw_post = posts_data[0][:500]  # Displaying first 500 characters for a clearer view
sample_raw_post

'https://ethresear.ch/t/octopus-contract-and-its-applications/17844'

New chat

Your expert on Ethereum's technology and developments